{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# max_depth 和 min_child_weight 粗调优，步长2，然后再采用步长为1进行细调优\n",
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "      <th>interest_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.5</td>\n",
       "      <td>3</td>\n",
       "      <td>3000</td>\n",
       "      <td>1200.0</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>-1.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>5465</td>\n",
       "      <td>2732.5</td>\n",
       "      <td>1821.666667</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.0</td>\n",
       "      <td>1425.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3275</td>\n",
       "      <td>1637.5</td>\n",
       "      <td>1637.500000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>18</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4</td>\n",
       "      <td>3350</td>\n",
       "      <td>1675.0</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>28</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 228 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathrooms  bedrooms  price  price_bathrooms  price_bedrooms  room_diff  \\\n",
       "0        1.5         3   3000           1200.0      750.000000       -1.5   \n",
       "1        1.0         2   5465           2732.5     1821.666667       -1.0   \n",
       "2        1.0         1   2850           1425.0     1425.000000        0.0   \n",
       "3        1.0         1   3275           1637.5     1637.500000        0.0   \n",
       "4        1.0         4   3350           1675.0      670.000000       -3.0   \n",
       "\n",
       "   room_num  Year  Month  Day       ...        walk  walls  war  washer  \\\n",
       "0       4.5  2016      6   24       ...           0      0    0       0   \n",
       "1       3.0  2016      6   12       ...           0      0    0       0   \n",
       "2       2.0  2016      4   17       ...           0      0    0       0   \n",
       "3       2.0  2016      4   18       ...           0      0    0       0   \n",
       "4       5.0  2016      4   28       ...           0      0    1       0   \n",
       "\n",
       "   water  wheelchair  wifi  windows  work  interest_level  \n",
       "0      0           0     0        0     0               1  \n",
       "1      0           0     0        0     0               2  \n",
       "2      0           0     0        0     0               0  \n",
       "3      0           0     0        0     0               2  \n",
       "4      0           0     0        0     0               2  \n",
       "\n",
       "[5 rows x 228 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# path to where the data lies\n",
    "dpath = './data/'\n",
    "train = pd.read_csv(dpath +\"RentListingInquries_FE_train.csv\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = train['interest_level']\n",
    "X_train = train.drop(['interest_level'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "第一轮参数调整得到的n_estimators最优值（267），其余参数继续默认值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': range(3, 10, 2), 'min_child_weight': range(1, 6, 2)}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#max_depth 建议3-10， min_child_weight=1／sqrt(ratio_rare_event) =5.5\n",
    "max_depth = range(3,10,2)\n",
    "min_child_weight = range(1,6,2)\n",
    "param_test2_1 = dict(max_depth=max_depth, min_child_weight=min_child_weight)\n",
    "param_test2_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.59891, std: 0.00303, params: {'max_depth': 3, 'min_child_weight': 1},\n",
       "  mean: -0.59901, std: 0.00314, params: {'max_depth': 3, 'min_child_weight': 3},\n",
       "  mean: -0.59889, std: 0.00274, params: {'max_depth': 3, 'min_child_weight': 5},\n",
       "  mean: -0.58831, std: 0.00460, params: {'max_depth': 5, 'min_child_weight': 1},\n",
       "  mean: -0.58809, std: 0.00429, params: {'max_depth': 5, 'min_child_weight': 3},\n",
       "  mean: -0.58804, std: 0.00431, params: {'max_depth': 5, 'min_child_weight': 5},\n",
       "  mean: -0.59218, std: 0.00531, params: {'max_depth': 7, 'min_child_weight': 1},\n",
       "  mean: -0.59161, std: 0.00435, params: {'max_depth': 7, 'min_child_weight': 3},\n",
       "  mean: -0.58957, std: 0.00496, params: {'max_depth': 7, 'min_child_weight': 5},\n",
       "  mean: -0.61048, std: 0.00481, params: {'max_depth': 9, 'min_child_weight': 1},\n",
       "  mean: -0.60373, std: 0.00324, params: {'max_depth': 9, 'min_child_weight': 3},\n",
       "  mean: -0.59851, std: 0.00357, params: {'max_depth': 9, 'min_child_weight': 5}],\n",
       " {'max_depth': 5, 'min_child_weight': 5},\n",
       " -0.5880408408348814)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb2_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=267,  #第一轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "gsearch2_1 = GridSearchCV(xgb2_1, param_grid = param_test2_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch2_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch2_1.grid_scores_, gsearch2_1.best_params_,     gsearch2_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([ 89.71415234,  89.70992022,  88.21081915, 140.30316167,\n",
       "        140.05321937, 141.44428811, 195.83496647, 196.47732067,\n",
       "        190.51155195, 249.35278845, 249.70134563, 193.96099286]),\n",
       " 'std_fit_time': array([ 2.03775328,  1.61779384,  2.92239145,  2.68470804,  2.77684545,\n",
       "         0.75587867,  2.77130974,  2.29488345,  4.80102359,  1.45703219,\n",
       "         3.54912597, 15.35640127]),\n",
       " 'mean_score_time': array([0.33225031, 0.32014604, 0.33657899, 0.57022347, 0.51257133,\n",
       "        0.56389971, 1.28645878, 1.25996642, 0.84803667, 2.94506803,\n",
       "        3.25873637, 0.74291239]),\n",
       " 'std_score_time': array([0.02437353, 0.01764851, 0.03379101, 0.1204808 , 0.04943019,\n",
       "        0.13160921, 0.54157323, 0.47717204, 0.20976125, 0.56736751,\n",
       "        1.68183513, 0.05026271]),\n",
       " 'param_max_depth': masked_array(data=[3, 3, 3, 5, 5, 5, 7, 7, 7, 9, 9, 9],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_min_child_weight': masked_array(data=[1, 3, 5, 1, 3, 5, 1, 3, 5, 1, 3, 5],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'max_depth': 3, 'min_child_weight': 1},\n",
       "  {'max_depth': 3, 'min_child_weight': 3},\n",
       "  {'max_depth': 3, 'min_child_weight': 5},\n",
       "  {'max_depth': 5, 'min_child_weight': 1},\n",
       "  {'max_depth': 5, 'min_child_weight': 3},\n",
       "  {'max_depth': 5, 'min_child_weight': 5},\n",
       "  {'max_depth': 7, 'min_child_weight': 1},\n",
       "  {'max_depth': 7, 'min_child_weight': 3},\n",
       "  {'max_depth': 7, 'min_child_weight': 5},\n",
       "  {'max_depth': 9, 'min_child_weight': 1},\n",
       "  {'max_depth': 9, 'min_child_weight': 3},\n",
       "  {'max_depth': 9, 'min_child_weight': 5}],\n",
       " 'split0_test_score': array([-0.59352036, -0.59392653, -0.59422693, -0.57991734, -0.58008502,\n",
       "        -0.58110462, -0.58233681, -0.58430269, -0.58061707, -0.60260422,\n",
       "        -0.59996625, -0.59370858]),\n",
       " 'split1_test_score': array([-0.59814972, -0.59752877, -0.59794935, -0.58810537, -0.58751246,\n",
       "        -0.58571772, -0.59198674, -0.58973838, -0.58853358, -0.607281  ,\n",
       "        -0.60091997, -0.59640496]),\n",
       " 'split2_test_score': array([-0.59964732, -0.5993134 , -0.59905928, -0.58914138, -0.58968314,\n",
       "        -0.58889427, -0.59403342, -0.59361342, -0.59093196, -0.6129138 ,\n",
       "        -0.60537573, -0.60096876]),\n",
       " 'split3_test_score': array([-0.60237895, -0.60301886, -0.60171548, -0.5906857 , -0.59116012,\n",
       "        -0.59099081, -0.5943543 , -0.59320138, -0.59268581, -0.61452905,\n",
       "        -0.60340331, -0.59753215]),\n",
       " 'split4_test_score': array([-0.60084701, -0.60125132, -0.60149023, -0.59370606, -0.59203182,\n",
       "        -0.59349844, -0.59817765, -0.59719587, -0.5950744 , -0.61505503,\n",
       "        -0.60898607, -0.60392988]),\n",
       " 'mean_test_score': array([-0.59890855, -0.59900764, -0.5988881 , -0.58831084, -0.58809427,\n",
       "        -0.58804084, -0.59217742, -0.59161001, -0.58956823, -0.61047634,\n",
       "        -0.60372994, -0.59850854]),\n",
       " 'std_test_score': array([0.00303174, 0.00313763, 0.00273505, 0.0046028 , 0.00428756,\n",
       "        0.00430629, 0.00531146, 0.00435073, 0.00496163, 0.00480648,\n",
       "        0.00324032, 0.0035737 ]),\n",
       " 'rank_test_score': array([ 9, 10,  8,  3,  2,  1,  6,  5,  4, 12, 11,  7], dtype=int32),\n",
       " 'split0_train_score': array([-0.57374091, -0.57478956, -0.57531246, -0.49997673, -0.5069548 ,\n",
       "        -0.51141261, -0.38460558, -0.41009076, -0.42459   , -0.2573902 ,\n",
       "        -0.30297876, -0.33323872]),\n",
       " 'split1_train_score': array([-0.5727064 , -0.57320099, -0.5739466 , -0.49876084, -0.5053716 ,\n",
       "        -0.5090058 , -0.38512469, -0.41011914, -0.42356005, -0.25371274,\n",
       "        -0.29876675, -0.33199698]),\n",
       " 'split2_train_score': array([-0.57267986, -0.57315613, -0.57405704, -0.49819873, -0.50433209,\n",
       "        -0.50928768, -0.38500553, -0.41057949, -0.42313926, -0.25612358,\n",
       "        -0.30526862, -0.33482261]),\n",
       " 'split3_train_score': array([-0.5710444 , -0.57183879, -0.57293821, -0.49631125, -0.50481812,\n",
       "        -0.50771991, -0.38349105, -0.40936047, -0.42377879, -0.25379545,\n",
       "        -0.30225257, -0.33333151]),\n",
       " 'split4_train_score': array([-0.57174308, -0.57244158, -0.5730319 , -0.49640116, -0.50315178,\n",
       "        -0.507337  , -0.38379359, -0.40572117, -0.42066713, -0.25333572,\n",
       "        -0.30225725, -0.33268166]),\n",
       " 'mean_train_score': array([-0.57238293, -0.57308541, -0.57385724, -0.49792974, -0.50492568,\n",
       "        -0.5089526 , -0.38440409, -0.40917421, -0.42314705, -0.25487154,\n",
       "        -0.30230479, -0.3332143 ]),\n",
       " 'std_train_score': array([0.00092065, 0.00098905, 0.00085923, 0.00140779, 0.00125114,\n",
       "        0.00143502, 0.00065237, 0.00177008, 0.00132679, 0.00159817,\n",
       "        0.0020862 , 0.00093451])}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch2_1.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.588041 using {'max_depth': 5, 'min_child_weight': 5}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAELCAYAAADz6wBxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl8VOX1+PHPmcm+QFiCLBFBBEFWIaKIIMFGsSoWiXzF6k/UatVaa22pdakWa61+aRVt/bpWq21dalhKLSqoQRAXBAyKoIIbhjUCgSRkn/P7496ESTJZJ5PJct6v15CZu565IXPmee695xFVxRhjjGkuT7gDMMYY075ZIjHGGBMUSyTGGGOCYonEGGNMUCyRGGOMCYolEmOMMUGxRGKMMSYolkiMMcYExRKJMcaYoESEO4DW0LNnTx0wYEC4wzDGmHZl/fr136lqckPLdYpEMmDAANatWxfuMIwxpl0RkW8as5x1bRljjAmKJRJjjDFBsURijDEmKJ3iHIkxpvHKysrIycmhuLg43KGYVhITE0NKSgqRkZHNWt8SiTGmmpycHBITExkwYAAiEu5wTIipKvv27SMnJ4eBAwc2axvWtWWMqaa4uJgePXpYEukkRIQePXoE1QK1RGKMqcWSSOcS7O/burbqkb9yJSJC/GmnIV5vuMMxxpg2yRJJPfY/9TSH164londvki6YQdcLZhKV0i/cYRljTJsS0q4tEZkmIp+JyDYR+XUdy8wSkc0i8omIPOc3/VURyRORl2ssP1BE3ne3+aKIRIUq/v5PPkG/hx4keshgvnvkUb5IT2f7FVdy6JVX8JWWhmq3xnRqeXl5/N///V+z1l2wYAGHDx9u4YhaxpQpU5pdYWPJkiVs3ry5SdsqLi5m/PjxjB49muHDh3PnnXc2a9+NEbJEIiJe4GHgbOAEYLaInFBjmcHALcBEVR0O3Og3ez5waYBN3wc8oKrHAQeAK0MQvhNfVBRdzjyT/o8/znFvvkHP639CyddfsePnN7Ft8uns+cO9lGzdGqrdG9MpddREEoyaiaQxoqOjefPNN9m4cSPZ2dm8+uqrvPfeeyGJL5RdW+OBbar6JYCIvACcD/gfjauAh1X1AICq7q2coapviMgU/w2Kc0ZoKnCxO+kZ4LfAI6F5C0dE9ulD8k9+Qs9rrqHw3ffIy8xk/3PPsf+ZZ4gdM4akCzPoMm0anvj4UIdiTKuZ959P2LzzUItu84S+XbjzvOF1zv/1r3/NF198wZgxY0hPT6dXr17861//oqSkhBkzZjBv3jwKCwuZNWsWOTk5VFRU8Jvf/IY9e/awc+dO0tLS6NmzJ1lZWQG3n5CQwLXXXsuyZcvo06cP99xzD7/61a/Yvn07CxYsYPr06Xz99ddceumlFBYWAvCXv/yFU089lcWLF/OXv/yF119/nd27d3P66aezatUqevfuXWs/RUVFXH755WzcuJGhQ4dSVFRUNW/58uXceeedlJSUMGjQIJ5++mkSEhIYMGAAs2bN4pVXXiE2NpbnnnuOvXv3snTpUt566y3uvvtuFi5cCMBLL73EddddR15eHn/961+ZNGlStf2LCAkJCYBzb1BZWVnILqIIZddWP+Bbv9c57jR/Q4AhIrJGRN4TkWkNbLMHkKeq5fVsM6TE6yXhtImkLHiAwW+tpNfNN1ORn8+u225n66TJ7PrNHRR99BGq2pphGdNh3HvvvQwaNIjs7GzS09PZunUra9euJTs7m/Xr17Nq1SpeffVV+vbty8aNG9m0aRPTpk3jhhtuoG/fvmRlZdWZRAAKCwuZOnUqn3zyCYmJidx+++2sWLGCxYsXc8cddwDQq1cvVqxYwYYNG3jxxRe54YYbAJgxYwZ9+vTh4Ycf5qqrrmLevHkBkwjAI488QlxcHFu2bGHevHmsX78egO+++467776b119/nQ0bNpCamsr9999ftV7Xrl35+OOPuf7667nxxhs59dRTmT59OvPnzyc7O5tBgwYBUF5eztq1a1mwYAHz5s0DYOfOnXz/+9+v2lZFRQVjxoyhV69epKenc/LJJwfxm6lbuE+2RwCDgSlACrBKREaqal6wGxaRq4GrAfr37x/s5gKK6N6dHpfPofucyyj6MJu8zEwOvvwyeS+9RPSQISRlZNB1+nl4k5JCsn9jQq2+lkNrWL58OcuXL+fEE08EoKCggK1btzJp0iR+8YtfcPPNN3PuuefW+jZen6ioKKZNc76zjhw5kujoaCIjIxk5ciRff/014HyDv/7668nOzsbr9fL5559Xrf/nP/+ZESNGcMoppzB79uw697Nq1aqqBDRq1ChGjRoFwHvvvcfmzZuZOHEiAKWlpUyYMKFqvcptzp49m5///Od1bv+CCy4AYNy4cVVx9+3bl2XLllUt4/V6yc7OJi8vjxkzZrBp0yZGjBjRqOPUFKFMJDuAo/1ep7jT/OUA76tqGfCViHyOk1g+qGOb+4AkEYlwWyWBtgmAqj4OPA6Qmpoa0uaBiBA39kTixp7IUbfewqH/LiMvM5M999zD3j/+kcT0dJIuzCBu/HjEY7fuGNNYqsott9zCj3/841rzNmzYwLJly7j99ts544wzqloTDYmMjKzq4vF4PERHR1c9Ly93OjseeOABjjrqKDZu3IjP5yMmJqZq/ZycHDweD3v27MHn8+Fp4t+0qpKens7zzz8fcL5/91N9XVGVcXu93qq465KUlERaWhqvvvpqSBJJKD/VPgAGu1dZRQEXAUtrLLMEpzWCiPTE6er6sq4NqtNflAVkuJMuA/7dsmEHx5uQQLf/mcXAl/7FwCWLSZo1i4LVq9k+53K+OGsa3z36GGV79ja8IWM6qcTERPLz8wE466yzeOqppygoKABgx44d7N27l507dxIXF8cll1zC3Llz2bBhQ611g3Hw4EH69OmDx+Ph73//OxUVFYDTnXTFFVfw/PPPM2zYsGpdUjVNnjyZ555zLkTdtGkTH330EQCnnHIKa9asYdu2bYDT1ebf4nnxxRerfla2VJrzvnJzc8nLczp3ioqKWLFiBUOHDm3SNhorZInEbTFcD7wGbAH+paqfiMhdIjLdXew1YJ+IbMZJEHNVdR+AiKwGXgLOEJEcETnLXedm4CYR2YZzzuSvoXoPwYoZOpTet9/G4FVv0Xf+fCL79iV3wQK2paXx7bXXkf/mm2gD3ySM6Wx69OjBxIkTGTFiBCtWrODiiy9mwoQJjBw5koyMDPLz8/n4448ZP348Y8aMYd68edx+++0AXH311UybNo20tLSgYrjuuut45plnGD16NJ9++inx7kU099xzD5MmTeK0007j/vvv58knn2TLli0Bt3HttddSUFDAsGHDuOOOOxg3bhwAycnJ/O1vf2P27NmMGjWKCRMm8Omnn1atd+DAAUaNGsWDDz7IAw88AMBFF13E/PnzOfHEE/niiy/qjNv/HMmuXbtIS0tj1KhRnHTSSaSnp3PuuecGdVzqIp3hpHBqaqq2lRESS7/5hryFi8hbvIiK3O+ISE6m64wZJM28gKhjjgl3eMawZcsWhg0bFu4wOqXK0Vx79uzZ6vsO9HsXkfWqmtrQutZh38qijjmGXjf9nMFZWaT838PEjBjBvr/+lS/OmsY3l83h4H9exldSEu4wjTGm0cJ91VanJRERJE6dSuLUqZTt2cvBJUvIy8xk59y5eO7uStfzziPpwgxijj8+3KEa0y6dfPLJlNT4Uvb3v/+dkSNHtuh+XnvtNW6++eZq0wYOHMjixYubvK3Kq6/aG+vaakPU5+Pw2rXkZS4kf/lytLSUmJEjScrIoMs538fr3lxkTChZ11bnZF1bHYR4PMSfcgr9/jifwave4qjbbkNLSth9551snTSZnbfexuENH9rNjsaYNsW6ttoob1IS3S+9hG6X/JDiTZvI+9dLHPrvfzm4aBFRgwY5NzueP52I7t3DHaoxppOzFkkbJyLEjhxJn9/dxeDVq+jz+7vxdunC3vvuY+vpU8i58ecUvL0G9fnCHaoxppOyFkk74omPJ2nmTJJmzqRk2zbyMhdycMkS8l99lci+fek68wKSLriAyD59wh2qMaYTsRZJOxV93HEc9eubOW7VW/Rb8ABRAwfy3V8eZtvUM9h+9dUcck/WG9PedNQy8q09Hgk496WMHDmSMWPGkJra4DnzZrNE0s55oqLoMm0a/f/6JINWrKDntddS8vlWdtzwM7amTWXP/PmUfPlVuMM0ptE6aiIJRnPGI6mUlZVFdnZ2s5NYY1jXVgcSldKP5Bt+Ss+fXEfhmjXkvZTJ/meeZf9fnyI2dZxzGfFZZ+GJjQ13qKa9eOXXsPvjlt1m75Fw9r11zrbxSFpmPJLWZC2SDki8XhImTyblzw8xeGUWveb+korv9rHr17c4Y6bMm0fRJ5+EO0xjArLxSFpuPBIR4cwzz2TcuHE8/vjjQfxW6mctkg4uomdPelx5Jd2vuIKi9evJeymTg4sWk/f8C0SfMMy5jPjcc/F26RLuUE1bVE/LoTXYeCTBjUfy9ttv069fP/bu3Ut6ejpDhw5l8uTJjTpOTWGJpJMQEeJSU4lLTeWo227l0H//S95Lmey563fsve9/6TLtLLrOnEncSSeFbDhOY5rKxiMJbjySfv2cAWR79erFjBkzWLt2bUgSiXVtdULeLl3oNns2AxctZMDCTJJmXkD+m1ls/3+X8eW0s/nuiScoz80Nd5imk7LxSFpmPJLCwsKqdQoLC1m+fHlIBrUCa5F0erHDhxM7fDi95s4lf/ly8l7KJPdP95P74EMkpk0hKSOD+NNOQ7zecIdqOgn/8UjOPvvsqvFIwDlR/o9//INt27Yxd+5cPB4PkZGRPPLII8CR8Ugqz5U013XXXcfMmTN59tlnmTZtWsDxSEaPHs1JJ53EOeecE7A22bXXXsvll1/OsGHDGDZsWMDxSCqLSt59990MGTIEODIeSXR0dFWr5aKLLuKqq67ioYceIjMzs864d+7cyY9+9COWLVvGnj17mDFjBuAkwIsvvriqS6+lWdFGU0vJl19xcNFC8hYvoWLfPiKOOoqkmRfQ9YILiEpJCXd4JsSsaGP42HgkpsOIPnYgvX75SwavzKLfnx8ieujxfPfoY3zxvXS2X3EFh5Ytw2c3OxpjXNa1ZeokkZF0SU+nS3o6Zbt2kbd4MQczF7Ljpl/gTUqi6/nTScrIIHrw4HCHakwtNh5J67GuLdMk6vNR+O675GVmkv/6G1BWRuzo0SRdmEGXs8/G4/Ylm/bLurY6p2C6tqxFYppEPB4SJk4kYeJEyvfv5+DSpeS9lMmu23/Dnnv+QJdzvk9SRgYxo0bZZcTGdBKWSEyzRXTvTo85c+h+2WUUZWeTl5nJwZed+1OiBw92WinnnUdEt27hDtUYE0J2st0ETUSIO/FE+v7+9wxevYred81DYmPZc88f2Db5dHbc9AsK333XxkwxpoOyRGJalDchgW6zZjHwXy8y8N9LSLroIgrWrGH75VfwxZln8d2jj1K2Z0+4wzRtWEet/tvaZeQ/++wzxowZU/Xo0qULCxYsaNb+G2KJxIRMzPHH0/u2Wxm86i36/vGPRKakkLvgQbalTeXba64l/4030LKycIdp2piOmkiC0Zwy8scffzzZ2dlVxS7j4uKqblBsaZZITMh5oqPpeu45HPO3pxm0/DV6XHUVxZ98Qs5Prmfr1Kns/dP9lH7zTbjDNG2Efxn5uXPnMn/+fE466SRGjRrFnXfeCTglP8455xxGjx7NiBEjePHFF3nooYeqysinpaXVuf2EhATmzp3L8OHD+d73vsfatWuZMmUKxx57LEuXLgWcy3AnTZrE2LFjGTt2LO+88w4Aixcv5owzzkBV2bVrF0OGDGH37t0B91NUVMRFF13EsGHDmDFjRq0y8hMmTGDs2LFceOGFVSVgBgwYwK9+9StGjhzJ+PHj2bZtG++88w5Lly5l7ty5jBkzhi+++AJwysiPHz+eIUOGsHr16nqP6RtvvMGgQYM45phjGvlbaBo72W5aVVT//vT6+Y0k//R6ClatJi8zk31PPcW+J54gbvx4ki7MIDE9HY9fkTwTPvetvY9P93/aotsc2n0oN4+/uc759957L5s2bSI7O5vly5eTmZnJ2rVrUVWmT5/OqlWryM3NpW/fvvz3v/8FnNpYXbt25f777ycrK6veO8Mry8jPnz+fGTNmVJWR37x5M5dddhnTp0+vKiMfExPD1q1bmT17NuvWrWPGjBksXLiQhx9+mFdffbXRZeQ/+ugjxo4dC1QvIx8fH899993H/fffX1V0srKM/LPPPsuNN97Iyy+/zPTp0zn33HPJyMio2n5lGflly5Yxb948Xn/99WolUvy98MIL9VYqDpYlEhMWEhFB4tQ0EqemUbZnLweXLCEvM5Odc3+Fp0sXup53HkkXZhAzdGi4QzVhZGXkgysjX7n9pUuX8oc//KHBY9NclkhM2EUe1YueP76aHlf9iMNrPyAvM5O8l17iwD//ScyIEc7IjueegzchIdyhdjr1tRxag5WRD66MPMArr7zC2LFjOeqoo5oUZ1PYORLTZojHQ/wpJ9Pvj/MZvOotjrrtNrS0lN2//S1bJ01m5y23cnjDBjpDNYbOzMrIt0wZ+UrPP/98SLu1wFokpo3yJiXR/dJL6HbJDynetIm8lzI59PLLHFy8mKhjj3VGdvzB+UR07x7uUE0LszLyLVNGHpwktWLFCh577LFmH4vGsFpbpt3wFRZy6NXXyMvMpOjDDyEyksSpU50xU06dYGOmtBCrtRU+7bWMvLVITLvhiY8naeYFJM28gJJt28jLXMjBJUvIf+01Ivr2IemCmSRdMIPIvn3DHaoxnYq1SEy75istpeDNN8l7KZNC91r/+NNOIykjg8S0KUhUVJgjbH86SoukPZaRD6dgWiSWSEyHUZqzg4OLFpG3aBHlu3fj7d6drj/4AUkZM4k+9thwh9dudJREYprGRkg0BohK6UfyDT/luDde5+jHHyNu3Dj2P/ssX37/HL7+4SXkLV6CrwOWzzAm3EKaSERkmoh8JiLbROTXdSwzS0Q2i8gnIvKc3/TLRGSr+7jMb/pKd5vZ7qNXKN+DaX/E6yVh8mRS/vwQg1dm0WvuL6nYt49dt9zC1smns+u3v6Vo0yd2GbExLSRkJ9tFxAs8DKQDOcAHIrJUVTf7LTMYuAWYqKoHKpOCiHQH7gRSAQXWu+secFf9oapaX5VpUETPnvS48kq6X3EFRevXk/dSJgeX/Ju8F14ketgwkjJm0vXcc/F27RruUI1pt0LZIhkPbFPVL1W1FHgBOL/GMlcBD1cmCFXd604/C1ihqvvdeSuAaSGM1XRwIkJcaip977uXwaveoveddyAi7Pnd3WydfDo7fvUrCt16TsaYpgllIukHfOv3Osed5m8IMERE1ojIeyIyrZHrPu12a/1GbDxX00TeLl3oNns2AxctZMDCTJJmXkBB1kq2/7/L+HLa2Xz3xBOU5+aGO8xOq6OWkW/t8UgAHnzwQUaMGMHw4cNDNhYJhP9kewQwGJgCzAaeEJGkBtb5oaqOBCa5j0sDLSQiV4vIOhFZl2sfCqYOscOH0/uOO5wxU+67l4jkZHL/dD9bp6Tx7fXXk79yJVpPHSPT8jpqIglGc8Yj2bRpE0888QRr165l48aNvPzyy1VlWVpaKBPJDuBov9cp7jR/OcBSVS1T1a+Az3ESS53rqmrlz3zgOZwutFpU9XFVTVXV1OTk5BZ4O6Yj88TG0vX88znmH3/n2GXL6HH5HIo+zCbnmmvZdsb32Pvgg5Tm5IQ7zE7BxiNpmfFItmzZwsknn0xcXBwRERGcfvrpLFq0qBm/kYaF8s72D4DBIjIQJwlcBFxcY5klOC2Rp0WkJ05X15fAF8A9ItLNXe5M4BYRiQCSVPU7EYkEzgVeD+F7MJ1Q9LED6fXLX5L8s5+Rv3KlM2bKY4+z75FHiT91AkkZGSR873t4OsHNjrvvuYeSLS07Hkn0sKH0vvXWOufbeCQtMx7JiBEjuO2229i3bx+xsbEsW7aM1NQGbwlplpAlElUtF5HrgdcAL/CUqn4iIncB61R1qTvvTBHZDFQAc1V1H4CI/A4nGQHcpar7RSQeeM1NIl6cJPJEqN6D6dwkMpIu6el0SU+nbNcu8hYv5mDmQnbc9Au8XbvS9Qfn03XmTGLcYnum5dl4JM0fj2TYsGHcfPPNnHnmmcTHxzNmzBi8IapHF9JaW6q6DFhWY9odfs8VuMl91Fz3KeCpGtMKgXEhCdaYekT26UPyddfR85prKHz3XfIyM9n/3PPsf+ZZYkePJunCDLqcfTYet0psR1Ffy6E12HgkwY1HcuWVV3LllVcCcOutt5KSktKkWBsr3CfbjWlXxOMhYeJEUh54gMGr3qLXr2+morCAXbf/hq2TJrPrN7+haONGu4w4CDYeScuNR7J3r3NHxfbt21m0aBEXX1zz7ELLsOq/xjRTRLdu9Jgzh+6XXUZRdjZ5mZkcfPm/5L2USfTgwU4r5bzziOjWreGNmSo2HknLjUcyc+ZM9u3bR2RkJA8//DBJSQ1dFNs8VrTRmBZUUVDAoWXLyMtcSPFHHyGRkSSmp5N0YQZxJ5+MNLEbJBysaGP42Hgkxhi8CQl0mzWLbrNmUfzZZ86YKUuXcmjZMiJTUpySLDNmEBnC8bONaW3WIjEmxHwlJeSveJ28zEwOv/ceeDwkTJ5M0oUZJEyejERGhjvEajpKi8TGI2kaG4+kAZZITFtRun07eQsXcXDRIspzc/F2707C6aeTkDaFhIkT28RVXx0lkZimsa4tY9qJqP796fXzG0n+6fUUrFrNoWXLyH/jDQ4uXoxERhJ3yikkTk0jYcoUIvv0CVucqlrvpaemYwm2QWEtEmPCTMvKOLzhQwqyssjPepOyb7YDED1sGIlpaSSkpREz/IRWO1H/1VdfkZiYSI8ePSyZdAKqyr59+8jPz2fgwIHV5lnXlh9LJKa9UFVKv/rKSSpvZlH04Yfg8xHRqxcJU6aQkDaF+AkT8PjdINfSysrKyMnJobi4OGT7MG1LTEwMKSkpRNY4X2eJxI8lEtNelR84QMFbb1GQtZLC1avxHT6MxMQQf+qpThfY6acTYUVJTYhYIvFjicR0BL7SUg6v/aCqC6x85y4AYkaNcpJKWhrRQ4ZYd5RpMZZI/FgiMR2NqlLy+edVXWDFbvmNyL59SXDPq8SNP6lTVCg2oWOJxI8lEtPRle3de6QL7J130OJiPPHxxJ92mnNp8emnW6kW02SWSPxYIjGdia+4mMJ336UgayUFWVnOsMEeD7EnnljVBRY1cKB1gZkGWSLxY4nEdFbq81H8yWb3vEoWJVu2ABB5TH8S06Y6XWDjxiIRdkuZqc0SiR9LJMY4ynbuJH/lSgqyVnL4vffQsjI8XbqQMHmy0wU2aRLeLl3CHaZpIyyR+LFEYkxtFQWFFL6zxukCe+stKvbvh4gI4lJTSUybQsLUqUQdfXS4wzRhZInEjyUSY+qnFRUUbfyo6tLi0m1fABB13KCqLrDY0aOQEA3VatomSyR+LJEY0zSl335bdWnx4XXroLy8TRaYNKFlicSPJRJjmq/i0CEK336b/DezKFi1Ct+hQ1UFJhPSppCYlhbWApMmdCyR+LFEYkzLqL/A5BQS0qa2aoFJE1qWSPxYIjGm5dVZYDI52b27PvQFJk1oWSLxY4nEmNCrVmDy7bfxFRZWFZhMSJtC4pQpVmCynbFE4scSiTGty7/AZEFWFmU7dwJugUn30mIrMNn2WSLxY4nEmPCpq8BkRN8+JE5JI2HqVCsw2Ua1WCIRkUFAjqqWiMgUYBTwrKrmtUikrcASiTFtR3luLgVvvUX+m1lHCkzGxRE/aZIVmGxjWjKRZAOpwABgGfBvYLiqfr8F4mwVlkiMaZvqLTBZeXe9FZgMm5ZMJBtUdayIzAWKVfXPIvKhqp7YUsGGmiUSY9q+egtMVnaBjT0RqTEcrAmdlkwk7wMLgNuA81T1KxHZpKojWibU0LNEYkz7U7ZrFwUrVzp31/sXmJw0iYSpaVZgshW0ZCI5AbgGeFdVnxeRgcAsVb2vZUINPUskxrRvvsJCCt55h4I3swIXmExLI6p//3CH2eGE5KotEekGHK2qHwUTXGuzRGJMx+FfYLJgZRYlW7cBlQUm09wCk6OtwGQLaMkWyUpgOhABrAf2AmtU9aYWiLNVWCIxpuMKWGCyWzenwOTUNCswGYSWTCQfquqJIvIjnNbInSLykaqOaqlgQ80SiTGdQ50FJk8+mYSpaVZgsolaMpF8DJwJPAPcpqofWCIxxrR1DReYTCNm+HArMFmPlkwkFwK/wenOulZEjgXmq+rMlgk19CyRGNO5VSswmZVF0Qa/ApNTppAwNc0KTAbQJkqkiMg04EHACzypqvcGWGYW8FtAgY2qerE7/TLgdnexu1X1GXf6OOBvQCzODZI/0wbehCUSY4y/8gMHKFy1yrm73r/A5IQJTheYFZgEWrZFkgL8GZjoTlqN8+Gd08B6XuBzIB3IAT4AZqvqZr9lBgP/Aqaq6gER6aWqe0WkO7AO5456xTnJP85dZi1wA/A+TiJ5SFVfqS8WSyTGmLpoaSmFH3zgXFrsX2By5EgSpzpXgUUff3ynvLu+JRPJCuA54O/upEuAH6pqegPrTQB+q6pnua9vAVDVP/gt87/A56r6ZI11ZwNTVPXH7uvHgJXuI0tVhwZari6WSIwxjeEUmNxKQdab5GdlUbyxRoHJtDTiTh7faQpMNjaRRDRiW8mq+rTf67+JyI2NWK8f8K3f6xzg5BrLDAEQkTU43V+/VdVX61i3n/vICTDdGGOCJiLEHD+EmOOH0POaa6oVmMxbtIgDzz3nFJg87TRn8K4pVmASGpdI9onIJcDz7uvZwL4W3P9gYAqQAqwSkZEtsWERuRq4GqC/3fFqjGmGiORkkjIySMrIcApMvvee0wW2ciX5y5c7BSbHjKnqAos69thO2QXWmERyBc45kgdwzle8A8xpxHo7gKP9Xqe40/zlAO+rahnwlYh8jpNYduAkF/91V7rTUxrYJgCq+jjwODhdW42I1xhj6uRPtpOXAAAYi0lEQVSJiSFxijPSo/p8FG/eQsGbb5K/Mou9f/wTe//4JyL796+6uz5u3NhOU2CyWVdticiNqrqggWUicE62n4HzYf8BcLGqfuK3zDScE/CXiUhP4ENgDEdOsI91F92Ac7J9f4CT7X9W1WX1xWLnSIwxoVRVYDIri8Pv1igwmZZGwuT2WWAypJf/ish2VW2wv0hEvo9TOdgLPKWqvxeRu4B1qrpUnDbgn4BpQAXwe1V9wV33CuBWd1O/rzxPIyKpHLn89xXgp3b5rzGmraizwOS4cUe6wNpJd3uoE8m3qnp0w0u2DZZIjDHhUGeByUGDqpJKWy4w2SZaJG2FJRJjTFtQVWAyK4vDH9QoMJmWRvzEiXgT2k6ByaATiYjk45yrqDULiFXVxpyobxMskRhj2ppqBSZXr8Z38OCRApNpU5wCk337hjXGNlEipa2wRNJ2qSqqzjcWn/vc5/6f9H+tgPpAqT7N567sU2eeT/22WWM5Z7OVyzRif/6x+RqxPxSfj/r357fdyv16RBBx7mHwiPua6q89HhCc5TwifutUf+2p2o6zjSPTj2zD47evavsM9Bq/GEQQD1XbDRRD5TZN02h5OYc3bKi6u770m28AiB46tKoLLBwFJi2R+GluIvl39g6+zC10mmWqNT483A8Eqn/IVJvu90Hk/+Ghfh9Elcv7/D5MqfahFGB/9X0ABtofzv6qvQcfNT4sa34oBthurf0DNT/YfZUflv7vS939B/pgNx3RkeRSMwE6P6nxus5E6vFLWtROnPUmQgIn0mqx1fyJu1y1ZFy5ncplA7+ulXw9gZKxs+2aMdQ8PrG7c+ix8X26bXyfxG1bEPVR1rUbB8ecQv7YUygcfiIaE9OoY3ja4J4kRDevA6kl72zvtBZ/uIOVn+UCuP8hqv/Cav5HrfyP7v+fs/KXWfVtker/efzXrfwDq7ldau6n2v6r/xFQ9a3UU+0P58gytbcReLs131eN9+ABqOd9Uf2PDKn+PiuPp9TYj8dT8/1Wfw813zP+HwBC3e/B/ZJc9/uq3H+g/R35XdX+Rh94f9Q4rnXtD6j1JcSntZO4T7Vai8antb/A1PfTFyCxa+U23S8FzvMAr3H25R9X5TYCxe3/pcV/3zW3A0fel/+XpbpeV3v/vhqvG3j/FT6lrOLIMQz0/qu/rt3y9fnqOoa1YwgYk9/vqnFOgOEnkDi4kJN2f8rJuz8hdc2b9HzrFYq9kWQnD+b93ifwfu8TOBBT96XFr990Osf1SmjsTpvFWiT18Pm06oPCGGNaQs1W/5HkUz2xojUTreIrK6N0/TpKV71F+dur0N27AJBhJ+A5dRIycTJ67HEoUrXdY5PjiYls3lVh1rXlx86RGGM6mtYoMNmS1X8DXb11EKfM+y9U9ctmR9lKLJEYYzq6qgKTWSspXLMGLS7GExfHMc/9k5ihQ5u1zZY8R7IApybWczjd2hcBg3DKljxF9ZpYxhhjwiBQgcnCVauIOvbYkO+7MS2Sjao6usa0bFUdE2heW2QtEmOMabrGtkgac1HyYRGZJSIe9zELKHbndfwTLMYYY+rVmETyQ+BSYK/7uBS4RERigetDGJsxxph2oMFzJO7J9PPqmP12y4ZjjDGmvWmwRSIiKSKyWET2uo+FIpLS0HrGGGM6h8Z0bT0NLAX6uo//uNOMMcaYRiWSZFV9WlXL3cffgOQQx2WMMaadaEwi2Scil4iI131cAuwLdWDGGGPah8YkkiuAWcBuYBeQAcwJYUzGGGPakQYTiap+o6rTVTVZVXup6g+Ama0QmzHGmHaguaOk3NSiURhjjGm3mptIrK66McYYoPmJxEqjGGOMAeq5s72O8vHgtEZiQxaRMcaYdqXORKKqia0ZiDHGmPapuV1bxhhjDGCJxBhjTJAskRhjjAmKJRJjjDFBsURijDEmKJZIjDHGBMUSiTHGmKBYIjHGGBMUSyTGGGOCYonEGGNMUCyRGGOMCUpIE4mITBORz0Rkm4j8OsD8OSKSKyLZ7uNHfvPuE5FN7uN//Kb/TUS+8ltnTCjfgzHGmPrVWbQxWCLiBR4G0oEc4AMRWaqqm2ss+qKqXl9j3XOAscAYIBpYKSKvqOohd5G5qpoZqtiNMcY0XihbJOOBbar6paqWAi8A5zdy3ROAVaparqqFwEfAtBDFaYwxJgihTCT9gG/9Xue402qaKSIfiUimiBztTtsITBOROBHpCaQBR/ut83t3nQdEJDok0RtjjGmUcJ9s/w8wQFVHASuAZwBUdTmwDHgHeB54F6hw17kFGAqcBHQHbg60YRG5WkTWici63NzckL4JY4zpzEKZSHZQvRWR4k6roqr7VLXEffkkMM5v3u9VdYyqpuOMyvi5O32XOkqAp3G60GpR1cdVNVVVU5OTk1vsTRljjKkulInkA2CwiAwUkSjgImCp/wIi0sfv5XRgizvdKyI93OejgFHAcv91RESAHwCbQvgejDHGNCBkV22parmIXA+8BniBp1T1ExG5C1inqkuBG0RkOlAO7AfmuKtHAqudXMEh4BJVLXfn/VNEknFaKdnANaF6D8YYYxomqhruGEIuNTVV161bF+4wjDGmXRGR9aqa2tBy4T7Zbowxpp2zRGKMMSYolkiMMcYExRKJMcaYoFgiMcYYExRLJMYYY4JiicQYY0xQLJEYY4wJiiUSY4wxQbFEYowxJiiWSIwxxgTFEokxxpigWCIxxhgTFEskxhhjgmKJxBhjTFAskRhjjAmKJRJjjDFBsURijDEmKCEbs70j2H5oO8UVxcR4Y4jyRhHtja56eD3ecIdnjDFtgiWSety79l5W71gdcF6EJ6JaYqn2iIh2Eo/HeV7fcv6vo7xRVUkrJqJ28orxxhDhiUBEWvlIGGNM3SyR1OPHo3/MD477ASUVJbUf5c7P0opSiiuKq/0sqSjhYPFBSny1lyupKMGnvmbHJMiRJBMgUQVKRoFaVJXrVVveG1N7W34JzSPWE2qMqc0SST1GJ4+G5JbfbpmvzEks5UcST33JqvJRLWmVF1Pqq72NovIi8kryjixffiTJlfnKgoo70hPZqGRUlYA8UQFbXk1tlUWItcKMacsskYRBpCeSSE8k8ZHxrbpfn/qqJ67KROWrnbSqJa/y2i2uQMmusKywzvUVbXbcHvHUSkBNbXnVl8QCJcLKZSyBGdMwSySdiEc8xETEEBMR06r7VVXKfeWUVJTUm4wCdRMWlxfX24VYUlFCQVkBJUWBt1HuKw8q9ihPVNDJqL7l4qPi6R3Xm7jIuBY62sa0PkskJuREhEhvJJHeSBJIaNV9V/gqKPWV1tniqkpkAc5n1UpaAbaRX5QfsNVWXFHcpDiTopPoE9+H3vG96RPfh74Jfaue94nvQ4/YHnaOyrRZlkhMh+b1eIn1xBIbEduq+1VVynxlDZ73OlR6iN2Fu9lduJudBTv5Nv9b1u5eS2FZYbXtRXoiqxJL7/je9E3oWy3x9Inv0+otTWMqWSIxJgREhChvFFHeKBJJbPL6+aX57CzYye7C3ewq3OU8Cpyf7+96n9yi3FpX/3WP6V4tsfSJ70OfhCPPu8d0t3M+JiQskRjTBiVGJXJ89+M5vvvxAeeX+crYe3hvVXLZXbibnYU72VW4i68Pfs07O9+hqLyo2jrR3uhaica/dXNU/FFEe6Nb4+2ZDsYSiTHtUKQnkn4J/eiX0C/gfFXlUOmhqpbMzsLqrZu3d7xNblFurfV6xPSodX7Gv1WTFJ1krRpTiyUSYzogEaFrdFe6RndlaPehAZcprShlz+E9VednKls2uwp3sfXAVlbnrK510UBsRGy1JFPzfE3vuN5EeiNb4y2aNsQSiTGdVJQ3iqMTj+boxKMDzldV8kryqp2f8T9f89n+z9hXvK/aOoKQHJtM74Qa52r8WjZdorpYq6aDsURijAlIROgW041uMd04occJAZcpqSg50mVWUP1czaf7PyVrexalvtJq68RFxDktGDfZ9I3vW+2y5+S4ZCI91qppTyyRGGOaLdobzTFdjuGYLscEnK+q7CveV5Vsal6Jtvm7zRwoOVBtHY94SI5NrnV+xv91YlTTr4QzoWOJxBgTMiJCz9ie9IztyYieIwIuU1ReVK1VU5lkdhfu5uPcj1nxzYpaFQoSIhMC3k9T+bpnbE8iPPbx1lrsSBtjwio2IpaBXQcysOvAgPN96mNf0T6nRVO4k90Fu488L9zNxtyNHCw5WG0dr3jpFdcrcKvGndbate46spAmEhGZBjwIeIEnVfXeGvPnAPOBHe6kv6jqk+68+4Bz3Om/U9UX3ekDgReAHsB64FJVrd4Ja4zpMDziITkumeS4ZEYljwq4zOGyw7UuBqh8nr03m9cKX6Ncq7dqEqMS6Rvv16JJqH6+Jjku2crSNFLIEomIeIGHgXQgB/hARJaq6uYai76oqtfXWPccYCwwBogGVorIK6p6CLgPeEBVXxCRR4ErgUdC9T6MMW1fXGQcg5IGMShpUMD5Fb4Kcotyq52fqTxfs7NwJ+v3rie/NL/aOhGeCI6KOyrg/TSVyceKbTpC2SIZD2xT1S8BROQF4HygZiIJ5ARglaqWA+Ui8hEwTUReAqYCF7vLPQP8Fkskxph6eD1e5z6X+N6MYUzAZQpKCwK2aHYX7uaDPR+w96u9tcrS+BfbDHS+pntM907RqgllIukHfOv3Ogc4OcByM0VkMvA58HNV/RbYCNwpIn8C4oA0nATUA8hzE0zlNgPf2muMMU2QEJXA4KjBDO42OOD8cl85uYdzq52fqUw43+Z/y/u73udw+eFq6/gX2/Rv1fSO713VjdYRim2G+2T7f4DnVbVERH6M08KYqqrLReQk4B0gF3gXqGjKhkXkauBqgP79+7ds1MaYTifCE+EkgoQ+jGVsrfmqSn5ZfsCbN3cV7uLdXe+Sezi31iBvlcU2/c/P+J+vaQ/FNkOZSHYA/rfMpnDkpDoAqup/W+yTwP/6zfs98HsAEXkOp8WyD0gSkQi3VVJrm37rPw48DpCamtr84fmMMaYRRIQuUV3o0r1L3cU2K8rYc3hPtXI0ledqvjz4JWt2rmlUsU3/8zVtodhmKBPJB8Bg9yqrHcBFHDm3AYCI9FHVXe7L6cAWd7oXSFLVfSIyChgFLFdVFZEsIAPnyq3LgH+H8D0YY0yLifRGkpKYQkpiSsD5qsrBkoMBWzS7C3ezesdqviv6rtZ6PWN71jkw2qCkQSFPNCFLJKpaLiLXA6/hXP77lKp+IiJ3AetUdSlwg4hMB8qB/cAcd/VIYLXbnDsEXOJ3XuRm4AURuRv4EPhrqN6DMca0JhEhKSaJpJgkhvUYFnCZ0opS9hTuqTpXU9W6KXCKba7KWUVJRUnV8kvOX1Ln1WwtFrdqx+/1SU1N1XXr1oU7DGOMCTlV5UDJgarWzGn9Tmv2CX0RWa+qqQ0tF+6T7cYYY1qQiNA9pjvdY7ozvMfwVtlnx7/A2RhjTEhZIjHGGBMUSyTGGGOCYonEGGNMUCyRGGOMCYolEmOMMUGxRGKMMSYodh9JfQr2QnkJeLwgXvBEgMfjPvef5oU2XlTNGGNCxRJJfZZcB9tWNHJhCZBcPM5PT4Rf8gl2WoTffvx+NmeaJ+LIfqpN81ZPmPVOqyuuJsZqidiYdssSSX0mXAcnTAdfBWgF+HzgK3ef1zetxnOtcJbx+Ro/zVfutIaqtu23XFOn1RiMp02Sxiaupk7zT2aNnVZHImx0sq4rQftP8/uygYCA+4+bVOXIz2ZPo45p0gLTCLxcS8RvXyraHUsk9Rk0NdwRtAxVJ5n4ymskuEDT6kqENZ83NcFV7i/QtJaOywflpbWnBVqu3mluDHT8enRtV0skPanKUy2b9BraXlNjpWnrNipW4AePQJL/iB4tzxJJZyBy5JuxaTrV2smlWsJsRCKsMwH6AHX24ezMfd6cadS/XK35jZ0WaNt1LdeU+Gnaup021qbE77ecz3dkuRCzRGJMQ0TAG4Hz5xLeAYSMaYvs8l9jjDFBsURijDEmKJZIjDHGBMUSiTHGmKBYIjHGGBMUSyTGGGOCYonEGGNMUCyRGGOMCYqohv6ux3ATkVzgm2au3hP4rgXDaSkWV9NYXE1jcTVNR43rGFVNbmihTpFIgiEi61Q1Ndxx1GRxNY3F1TQWV9N09risa8sYY0xQLJEYY4wJiiWShj0e7gDqYHE1jcXVNBZX03TquOwciTHGmKBYi8QYY0xQLJEAIvKUiOwVkU11zBcReUhEtonIRyIyto3ENUVEDopItvu4o5XiOlpEskRks4h8IiI/C7BMqx+zRsbV6sdMRGJEZK2IbHTjmhdgmWgRedE9Xu+LyIA2EtccEcn1O14/CnVcfvv2isiHIvJygHmtfrwaGVdYjpeIfC0iH7v7XBdgfmj/HlW10z+AycBYYFMd878PvIIzluUpwPttJK4pwMthOF59gLHu80Tgc+CEcB+zRsbV6sfMPQYJ7vNI4H3glBrLXAc86j6/CHixjcQ1B/hLa/8fc/d9E/BcoN9XOI5XI+MKy/ECvgZ61jM/pH+P1iIBVHUVsL+eRc4HnlXHe0CSiPRpA3GFharuUtUN7vN8YAvQr8ZirX7MGhlXq3OPQYH7MtJ91Dw5eT7wjPs8EzhDpGqg7nDGFRYikgKcAzxZxyKtfrwaGVdbFdK/R0skjdMP+NbvdQ5t4APKNcHtmnhFRIa39s7dLoUTcb7N+gvrMasnLgjDMXO7Q7KBvcAKVa3zeKlqOXAQ6NEG4gKY6XaHZIrI0aGOybUA+BXgq2N+WI5XI+KC8BwvBZaLyHoRuTrA/JD+PVoiad824JQwGA38GVjSmjsXkQRgIXCjqh5qzX3Xp4G4wnLMVLVCVccAKcB4ERnRGvttSCPi+g8wQFVHASs40goIGRE5F9irqutDva+maGRcrX68XKep6ljgbOAnIjK5lfYLWCJprB2A/zeLFHdaWKnqocquCVVdBkSKSM/W2LeIROJ8WP9TVRcFWCQsx6yhuMJ5zNx95gFZwLQas6qOl4hEAF2BfeGOS1X3qWqJ+/JJYFwrhDMRmC4iXwMvAFNF5B81lgnH8WowrjAdL1R1h/tzL7AYGF9jkZD+PVoiaZylwP9zr3w4BTioqrvCHZSI9K7sFxaR8Ti/z5B/+Lj7/CuwRVXvr2OxVj9mjYkrHMdMRJJFJMl9HgukA5/WWGwpcJn7PAN4U92zpOGMq0Y/+nSc804hpaq3qGqKqg7AOZH+pqpeUmOxVj9ejYkrHMdLROJFJLHyOXAmUPNKz5D+PUa01IbaMxF5Hudqnp4ikgPciXPiEVV9FFiGc9XDNuAwcHkbiSsDuFZEyoEi4KJQ/zG5JgKXAh+7/esAtwL9/WILxzFrTFzhOGZ9gGdExIuTuP6lqi+LyF3AOlVdipMA/y4i23AusLgoxDE1Nq4bRGQ6UO7GNacV4gqoDRyvxsQVjuN1FLDY/X4UATynqq+KyDXQOn+Pdme7McaYoFjXljHGmKBYIjHGGBMUSyTGGGOCYonEGGNMUCyRGGOMCYolEmOMMUGxRGJMG+GWAm/WXfZu+fK+LbEtY5rKEokxHcMcoG9DCxkTCpZIjKlBRAaIyKci8jcR+VxE/iki3xORNSKyVUTGu493xRng6B0ROd5d9+ci8pT7fKSIbBKRuDr200NEloszqNSTOGNFVM67RJxBp7JF5DH37nNEpEBEHnDXecMtc5IBpAL/dJePdTfzUxHZIM6AR0NDecxM52aJxJjAjgP+BAx1HxcDpwG/xCm78ikwSVVPBO4A7nHXexA4TkRmAE8DP1bVw3Xs407gbVUdjlNorz+AiAwD/geY6FbmrQB+6K4Tj1OOYzjwFnCnqmYC64AfquoYVS1yl/3OrQj7iBu3MSFhtbaMCewrVf0YQEQ+Ad5QVRWRj4EBONVmnxGRwThjQVTWQPOJyBzgI+AxVV1Tzz4mAxe46/1XRA6408/AqRr7gVs/KRZnvBBwxsF40X3+DyBQ5eVKlfPWV+7HmFCwRGJMYCV+z31+r304fze/A7JUdYY4g2it9Ft+MFBA889ZCPCMqt7SiGXrK5ZXGXMF9rduQsi6toxpnq4cGc9hTuVEEekKPITT2ujhnr+oyyqcLjNE5Gygmzv9DSBDRHq587qLyDHuPA9OBWPcdd92n+fjjFNvTKuzRGJM8/wv8AcR+ZDq3/YfAB5W1c+BK4F7KxNCAPOAyW7X2QXAdgBV3QzcjjN06kc4I+1VjnNRiDOS4SZgKnCXO/1vwKM1TrYb0yqsjLwx7YiIFKhqQrjjMMaftUiMMcYExVokxoSYiFwO/KzG5DWq+pNwxGNMS7NEYowxJijWtWWMMSYolkiMMcYExRKJMcaYoFgiMcYYExRLJMYYY4Ly/wHMSaWEQIMXzQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch2_1.best_score_, gsearch2_1.best_params_))\n",
    "test_means = gsearch2_1.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch2_1.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch2_1.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch2_1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch2_1.cv_results_).to_csv('my_preds_maxdepth_min_child_weights_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(max_depth), len(min_child_weight))\n",
    "train_scores = np.array(train_means).reshape(len(max_depth), len(min_child_weight))\n",
    "\n",
    "for i, value in enumerate(max_depth):\n",
    "    pyplot.plot(min_child_weight, -test_scores[i], label= 'test_max_depth:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'max_depth' )                                                                                                      \n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig('max_depth_vs_min_child_weght_1.png' )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "初步最佳参数：{'max_depth': 5, 'min_child_weight': 5}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': [4, 5, 6], 'min_child_weight': [4, 5, 6]}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# max_depth 和 min_child_weight 采用步长为1进行细调优\n",
    "max_depth = [4,5,6]\n",
    "min_child_weight = [4,5,6]\n",
    "param_test2_2 = dict(max_depth=max_depth, min_child_weight=min_child_weight)\n",
    "param_test2_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.59138, std: 0.00355, params: {'max_depth': 4, 'min_child_weight': 4},\n",
       "  mean: -0.59182, std: 0.00352, params: {'max_depth': 4, 'min_child_weight': 5},\n",
       "  mean: -0.59130, std: 0.00361, params: {'max_depth': 4, 'min_child_weight': 6},\n",
       "  mean: -0.58836, std: 0.00462, params: {'max_depth': 5, 'min_child_weight': 4},\n",
       "  mean: -0.58804, std: 0.00431, params: {'max_depth': 5, 'min_child_weight': 5},\n",
       "  mean: -0.58832, std: 0.00408, params: {'max_depth': 5, 'min_child_weight': 6},\n",
       "  mean: -0.58834, std: 0.00429, params: {'max_depth': 6, 'min_child_weight': 4},\n",
       "  mean: -0.58814, std: 0.00350, params: {'max_depth': 6, 'min_child_weight': 5},\n",
       "  mean: -0.58823, std: 0.00404, params: {'max_depth': 6, 'min_child_weight': 6}],\n",
       " {'max_depth': 5, 'min_child_weight': 5},\n",
       " -0.5880408408348814)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb2_2 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=267,  #第一轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch2_2 = GridSearchCV(xgb2_2, param_grid = param_test2_2, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch2_2.fit(X_train , y_train)\n",
    "\n",
    "gsearch2_2.grid_scores_, gsearch2_2.best_params_,     gsearch2_2.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([109.38590493, 108.60296698, 111.31884713, 136.33431425,\n",
       "        136.24213638, 136.81921816, 162.33939242, 158.40642118,\n",
       "        126.15608764]),\n",
       " 'std_fit_time': array([ 2.64040511,  1.75625266,  2.97570672,  3.86235869,  5.41760723,\n",
       "         3.53715582,  1.94150971,  6.57099738, 12.08033213]),\n",
       " 'mean_score_time': array([0.36454673, 0.4022099 , 0.4475585 , 0.56312857, 0.53334489,\n",
       "        0.47809358, 0.56151342, 0.53993516, 0.41669059]),\n",
       " 'std_score_time': array([0.04068625, 0.02282808, 0.09290005, 0.09328188, 0.03537572,\n",
       "        0.01051342, 0.00991181, 0.0920953 , 0.00509994]),\n",
       " 'param_max_depth': masked_array(data=[4, 4, 4, 5, 5, 5, 6, 6, 6],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_min_child_weight': masked_array(data=[4, 5, 6, 4, 5, 6, 4, 5, 6],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'max_depth': 4, 'min_child_weight': 4},\n",
       "  {'max_depth': 4, 'min_child_weight': 5},\n",
       "  {'max_depth': 4, 'min_child_weight': 6},\n",
       "  {'max_depth': 5, 'min_child_weight': 4},\n",
       "  {'max_depth': 5, 'min_child_weight': 5},\n",
       "  {'max_depth': 5, 'min_child_weight': 6},\n",
       "  {'max_depth': 6, 'min_child_weight': 4},\n",
       "  {'max_depth': 6, 'min_child_weight': 5},\n",
       "  {'max_depth': 6, 'min_child_weight': 6}],\n",
       " 'split0_test_score': array([-0.58551526, -0.58643709, -0.58554329, -0.57999297, -0.58110462,\n",
       "        -0.5808084 , -0.5809906 , -0.5830514 , -0.5818713 ]),\n",
       " 'split1_test_score': array([-0.59029249, -0.59033268, -0.58950023, -0.58762309, -0.58571772,\n",
       "        -0.58751973, -0.58697829, -0.58667782, -0.58630058]),\n",
       " 'split2_test_score': array([-0.5909616 , -0.59096431, -0.59130793, -0.58919946, -0.58889427,\n",
       "        -0.59015894, -0.58875024, -0.58772771, -0.58837589]),\n",
       " 'split3_test_score': array([-0.59516054, -0.59537926, -0.59466678, -0.5917878 , -0.59099081,\n",
       "        -0.59056768, -0.59161925, -0.58952047, -0.5908595 ]),\n",
       " 'split4_test_score': array([-0.59496757, -0.59599198, -0.59546555, -0.59320688, -0.59349844,\n",
       "        -0.59254809, -0.59335003, -0.5937178 , -0.59373744]),\n",
       " 'mean_test_score': array([-0.59137928, -0.59182081, -0.5912965 , -0.58836175, -0.58804084,\n",
       "        -0.58832031, -0.58833738, -0.5881387 , -0.5882286 ]),\n",
       " 'std_test_score': array([0.00354728, 0.00352115, 0.00360734, 0.00461511, 0.00430629,\n",
       "        0.0040835 , 0.00428691, 0.00349896, 0.00403532]),\n",
       " 'rank_test_score': array([8, 9, 7, 6, 1, 4, 5, 2, 3], dtype=int32),\n",
       " 'split0_train_score': array([-0.54409325, -0.5453562 , -0.54576414, -0.50879512, -0.51141261,\n",
       "        -0.51274299, -0.4662396 , -0.46921515, -0.47371905]),\n",
       " 'split1_train_score': array([-0.54398424, -0.54496901, -0.54558212, -0.5069972 , -0.5090058 ,\n",
       "        -0.51158429, -0.46475022, -0.46964231, -0.47125673]),\n",
       " 'split2_train_score': array([-0.54301959, -0.54373729, -0.54400307, -0.50747723, -0.50928768,\n",
       "        -0.51138489, -0.46653814, -0.47008479, -0.47169503]),\n",
       " 'split3_train_score': array([-0.54348513, -0.54395187, -0.5443394 , -0.50680502, -0.50771991,\n",
       "        -0.50938917, -0.46322323, -0.46913419, -0.47049845]),\n",
       " 'split4_train_score': array([-0.5428296 , -0.54375707, -0.54522883, -0.50425439, -0.507337  ,\n",
       "        -0.50818863, -0.46208091, -0.46653911, -0.47092367]),\n",
       " 'mean_train_score': array([-0.54348236, -0.54435429, -0.54498351, -0.50686579, -0.5089526 ,\n",
       "        -0.51065799, -0.46456642, -0.46892311, -0.47161858]),\n",
       " 'std_train_score': array([0.00050306, 0.00067542, 0.0006934 , 0.00147896, 0.00143502,\n",
       "        0.00163959, 0.00171481, 0.00123936, 0.00112128])}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch2_2.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.588041 using {'max_depth': 5, 'min_child_weight': 5}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaEAAAELCAYAAABwLzlKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xd4XNWZ+PHvOyojq4ysLrkXbGxTbAcFYhwDobelbBwWCPmZBC9pGxKSZWMWdqlJtMSbELIbAhsCpLCBkOawTmLjEkookR0MuABuGMtytzWSbPX398c9I42kGfXRlez38zzz3Dvnnnvm3LGsV+fcc88RVcUYY4zxQ8DvChhjjDl+WRAyxhjjGwtCxhhjfGNByBhjjG8sCBljjPGNBSFjjDG+sSBkjDHGNxaEjDHG+MaCkDHGGN8k+12BoS4/P18nTJjgdzWMMWbYWLNmzX5VLehJXgtC3ZgwYQLl5eV+V8MYY4YNEXm/p3mtO84YY4xvLAgZY4zxjW9BSERyRWS5iLzntjlx8jWLyBvutSQq/TwRWevSXxKRE1z6OBFZJSJ/E5E3ReTSqHNuF5HNIvKOiFyU+Ks0xhjTFT9bQouAFao6BVjh3sdyVFVnudcVUekPA59U1VnAU8CdLv1O4BlVnQ1cC/wAQERmuPcnARcDPxCRpIG+KGOMMT3nZxC6EnjS7T8JXNXL8xUIuf1sYFc36VcCv1DVelXdBmwGTu9DvY0xxgwQP0fHFalqpdvfDRTFyZcmIuVAE1Cmqr916QuBpSJyFAgDH3HpdwPLRORLQAZwvksfDbwaVe5Ol2aMMcYnCW0JicjzIvJ2jNeV0fnUW9413hKv41W1FLgeeFBEJrv0W4FLVXUM8DjwHZd+HfCES78U+KmI9Oo6ReRmESkXkfJ9+/b15lRjjDG9kNCWkKqeH++YiOwRkRJVrRSREmBvnDIq3HariKwGZotIGJipqq+5bE8Df3T7N+Hd80FVXxGRNCAfqADGRhU9xqXF+sxHgUcBSktLbf1zMzQ0HIHqSghXQHgXaAtkFkFWsbcdkQsBG/Bqhhc/u+OWAAuAMrf9XccMbsTcEVWtF5F8YC7wAHAIyBaRqar6LnABsNGdtgM4D3hCRKYDacA+93lPich3gFHAFOD1BF6fMT3XUOsFlkiAqapo2w/vgvBOOHqo6zICyZBRCFlFXlBqDVCFkOkCVeRYcnBwrsuYbvgZhMqAZ0TkJuB94BoAESkFPqeqC4HpwCMi0oLXdVimqhtcvn8EfuWOHQI+48r9GvA/InIrXhffja67b72IPANswLu/9EVVbR6kazXHs/rq7gNMXVXn89LzITQKRo6FcWd4+6HR7jXKy1OzF2p2e9vq3VCzx3tVVUDFWqjdR8ye7rSR7QNUa+ByaZFjaSNBJKFfjzm+iff72cRTWlqqNm2PiauuqpsAUwH14c7nZRR6gSR7jAsuHQJMVgmkpPW/fs1NcGR/+wBV7bY1u6P290BTXefzk9NcoOrYuipq37LKKIQkmwXMeERkjbuX3y37qTEmFlWoO9w+mMQKMg3VHU4U75dyaBTknwCTzo4KMG6bVQLJqYNzHUnJXtDIKu46n6oXLKv3xG5Z1eyBA1vg/ZfjdAsKZOR3CFZxAlcwMyGXaoYnC0Lm+KPq/SJtDSZxAkxjbfvzJOB1V4VGQeE0OOG8zq2YrGJIShnwKrdoCzWNNYTrw1Q1VBGuD9OszQSTgqQmpbbbtksLpCI96U4TgbRs71Uwteu8TfWuG9B1BVbv7twtuO8dL3C1NHY+PyXDBagOXX8duwXT82ygxXHAgpA5tqjCkYPefZauAkzT0fbnSZLXQgmNgqKTYOpFnQNMZlG/upxUlbrmunaBJLINN4Spqq8i3BDunN5QRXVDNS3a0qfPTQmkxA9SHbeB1C7zRfZb07PzCOaO8t4HOpwjyaQ21JJ85EDs1lX1HtizHrasjN1lKUkuOHUMUDFaVwPRdWl8YUHIDB8tLXDkQFSAcUGmqkOLprm+/XmBZMhyAaVkJky7tH33WGiU94ss0LNZnJpamuIGjejgESu9oaUhbrkBCRBKDRFKDZEdzCY7mM3Y0NjW99HbUGqI5EAyDc0NNLQ0UN9cT0Nz+23HtE7HW7z9msYaGuo6l9HQ3EBdc4z7RL2UJEmdg1lKkNSCTILFeaQmzSIoyQRVSWlpItjcRGpTI8GmeoKNR0ltOELw6DZSD71JsL6GoDaTohBUJahKqirBlAxSR+QSTM8nmF5AamYxwaxigpmjSA6VIK3D2HNsoMUQY0HIDA0tLd5Irq4CTHUlNHf4JR5IaQsmo0+DGVdEBRiXnlHYqVtHVb3urYYwVfWHCe/Z0RpUooNL9PvItrZjN10HGSkZ7QLGpJGTvMARDMUMKJFtRkoGgd49V51wqkpTS1OnoNYuYLXECHBxgmJjS2PMfNXN1Z3SIq8mbYIUICUFsmLOc+w0ALvg6C44us57MMMJtrSQqhBECRIgNZBMMJBCalKQYFIaqSkjSE3JIJiaQWpqiGBaNsHkEfFbgoHYLcmOaZH9ofbvOpRYEDKJ19LsdcWEK+Lch9kF1bugpan9eUnBtkAy9oyYo8nqUjMJN9V0ChRV9QcJ791O1QcuoHQIKtUN1TR3MUI/JZDitUZSswkFQxSnFzM1Z2q3wSQrNYuUwMDfE/KLiJCSlEJKUgqZ+DOgoKmlqcuWXMyWXlMd9Q3VNBw9SP3RQzTUHfbeN1TT0FhLfeMR6hvraairob6libAI9SI0iFAf8PYbJeDS+n8NyYHk2F2cgdhBq7sAF+li7a6bNZgUJCUphWRJ7tm9QR9YEDL909zk9e+HYwxNbg0wldDxF35yWmuLpWncHKoz8wln5FCVFiIcHEFVciph1OvGag0uBwkf2E64si3Y1HfseosSkABZqVlegHDBZEzmmE5BJBRsOx7ZpiWlDdn/tMeb5EAyyYFk0lPSE/MBTQ1eKzzOfauWmt001uylvnYvDdpIfXTAEqEhZQQNI0ZSP2IkDWnZ1KdlUZ+aQUNqOvUpI2hICVKflEp9IImGlsaY3adHmo5wuP5wp1ZkY0sjdU11aNxZzXomIIG4gS86WEWnjQyO5GulXxugLzk+C0ImvuZG7z9k5IHKWAGmZjdoCwrUihBOClCVmk44s5CqjBzCo6dSlTqTcGqQqkASYVHC2kS46QhV9WHCDQeoqXkfauJXIz05vV2AmBCa0O59vECSmZJp3SCme8mpkD3ae8UQAIJ496A4eijG81Z72gLXwd1Qsy72w8cSgIyCDkPYR8HIGMPZU0a0nqaqNGlTj+/5dXd/MNIl2jFfTWNNu7Ss1CwLQiaBmhrcPGS72rVi6qs+IFxdQVXNHsJ1B6lKChAOBKgKuG1KkHAwg3BqkHBRDuHiHKq0iXBzPc1Ej95qBPZC3V6o8/6azU7Nbm19FKYXccLIKe1aIzFbJ6khUhIw5NmYXhOB9FzvVTi967yNR12A6tiyihrOvudtbz9Wt3Aw1BqUJKuIlMxiUjILyeg4DVN67rAfaGFB6BjU3HCE6oPvET60larD2wlX76SqZjfho/upqjtEuKGaquY6L6i4IOO9kqgLCKQCuSl0XF1DELJSs9qN0BoV5yZ7x6AyInmEdW+Z40fKCMiZ4L260tLsPVLQbvaKDt2Cu/7mHYs1ICaQ4oJVF89bRWa8GKwHpHvJgtAQpaocaTrSeZjvkf1UhXcSrt1N1ZF9hOsOUdUQJtx0hHBzA2GaqQ508cs+GUYkBwklhchOySCUms34EbmE0gvJTs+Pe68klBoiKzXLureMGUiBJMgs8F7Fp3Sdt76mc/dfdLfg4R3wweveYwyx7iGNyOn+eausIq8VNoh/MFoQSrCG5oaYw3xjPUNSVXeIcN0hwg1VhBuP0ET8hxOTVQm1tJDd3EKIAAVJQSYnZ5KdGiKUlkt2RiGhzGKyQ+MIjRxPKKO4NahY95Yxw1Aw03vlTe46X3OjG2gR575VzR7Y8Yp3LNbAnuQRXuspdxL8v992Pj7ALAglgKpyya8v4WDdQY52fDI/igCZBMhuUUJNTWQ3N1DS3OIFlxa3TUonlJZDdnoBocwSskNjCGVPYMTI8cjIsd5T/qkJGjVkjBl+klLaHmPoSmR+xNauv8jUSy54DVJryIJQAogqZzdCSnMaoYZmso9WEaqv9QJLc1uAyRyRR1LH2ZMj22w30WXUKBljjBkwIl4X3YgcKDjRt2pYEEqEQIDbDxzyAkhoGoyOE2BsYTFjzHHOglCifPG17vMYY8xxzoY6GWOM8Y0FIWOMMb7xLQiJSK6ILBeR99w25vS4ItIsIm+415Ko9PNEZK1Lf0lETnDp40RklYj8TUTeFJFLXfoEETkaVdYPB+dKjTHGxONnS2gRsEJVpwAr3PtYjqrqLPe6Iir9YeCTqjoLeAq406XfCTyjqrOBa4EfRJ2zJaqszw3o1RhjjOk1P4PQlcCTbv9J4Kpenq9AyO1nA7u6STfGGDPE+Dk6rkhVK93+bjpOVNYmTUTKgSagTFUjj/AuBJaKyFEgDHzEpd8NLBORLwEZwPlRZU0Ukb+5/Heq6ouxPlBEbgZuBhg3blxfrs0YY0wPJLQlJCLPi8jbMV5XRudTVSXmZEcAjFfVUuB64EERicxZcStwqaqOAR4HvuPSrwOecOmXAj8VkQBQCYxz3XRfBZ4SkRAxqOqjqlqqqqUFBQV9/wKMMcZ0KaEtIVU9P94xEdkjIiWqWikiJcDeOGVUuO1WEVkNzBaRMDBTVSMP4zwN/NHt3wRc7M55RUTSgHxV3QvUu/Q1IrIFmAqU9/c6jTHG9I2f94SWAAvc/gLgdx0ziEiOiATdfj4wF9gAHAKyRWSqy3oBsNHt7wDOc+dMB9KAfSJSICJJLn0SMAXYmoDrMsYY00N+3hMqA54RkZuA94FrAESkFPicqi4EpgOPiEgLXsAsU9UNLt8/Ar9yxw4Bn3Hlfg34HxG5Fa+L70ZVVRE5C7hXRBqBFvcZBwfrYo0xxnQm3u0YE09paamWl1uPnTHG9JSIrHH38rtlMyYYY4zxjQUhY4wxvrEgZIwxxjcWhIwxxvjGgpAxxhjfWBAyxhjjGwtCxhhjfGNByBhjjG8sCBljjPGNBSFjjDG+sSBkjDHGNxaEjDHG+MaCkDHGGN9YEDLGGOMbC0LGGGN8Y0HIGGOMbywIGWOM8Y1vQUhEckVkuYi857Y5cfI1i8gb7rUkKv08EVnr0l8SkRNc+ngRWSEib4rIahEZE3XOAvd574nIgsRfpTHGmK742RJaBKxQ1SnACvc+lqOqOsu9rohKfxj4pKrOAp4C7nTpi4GfqOqpwL3At8ALesBdwBnA6cBd8QKfMcaYweFnELoSeNLtPwlc1cvzFQi5/Wxgl9ufAax0+6vc5wBcBCxX1YOqeghYDlzch3obY4wZIH4GoSJVrXT7u4GiOPnSRKRcRF4VkehAtRBYKiI7gU8BZS59HfD3bv9qIEtE8oDRwAdR5+90aZ2IyM3uM8v37dvX6wszxhjTMwkNQiLyvIi8HeN1ZXQ+VVW8lk0s41W1FLgeeFBEJrv0W4FLVXUM8DjwHZf+z8DZIvI34GygAmjuTb1V9VFVLVXV0oKCgt6caowxpheSE1m4qp4f75iI7BGRElWtFJESYG+cMircdquIrAZmi0gYmKmqr7lsTwN/dPl24VpCIpIJfFxVD4tIBXBOVNFjgNX9uDxjjDH95Gd33BIgMkJtAfC7jhlEJEdEgm4/H5gLbAAOAdkiMtVlvQDYGMknIpHruh34sdv/E3ChKzMHuNClGWOM8UlCW0LdKAOeEZGbgPeBawBEpBT4nKouBKYDj4hIC17ALFPVDS7fPwK/cscOAZ9x5Z4DfEtEFHgB+CKAqh4UkfuAv7p896rqwcRfpjHGmHjEux1j4iktLdXy8nK/q2GMMcOGiKxx9/K7ZTMmGGOM8Y0FIWOMMb6xIGSMMcY3FoSMMcb4xoKQMcYY31gQMsYY4xsLQsYYY3xjQcgYY4xvLAgZY4zxjQUhY4wxvrEgZIwxxjcWhIwxxvjGgpAxxhjfWBAyxhjjGwtCxhhjfGNByBhjjG8sCBljjPGNL0FIRHJFZLmIvOe2OXHyNYvIG+61JCr9PBFZ69JfEpETXPp4EVkhIm+KyGoRGdNdWcYYY/yT7NPnLgJWqGqZiCxy778eI99RVZ0VI/1h4EpV3SgiXwDuBG4EFgM/UdUnReRc4FvAp7opy5hjSmNjIzt37qSurs7vqphjXFpaGmPGjCElJaXPZfQqCLkWy1hVfbPPn+i5EjjH7T8JrCZ2EIpHgZDbzwZ2uf0ZwFfd/irgt/2ppDHD0c6dO8nKymLChAmIiN/VMccoVeXAgQPs3LmTiRMn9rmcbrvjXLdWSERygbXA/4jId/r8iZ4iVa10+7uBojj50kSkXEReFZGrotIXAktFZCdeS6fMpa8D/t7tXw1kiUheN2UZc0ypq6sjLy/PApBJKBEhLy+v3y3untwTylbVMN4v95+o6hnA+T2o4PMi8naM15XR+VRV8Vo2sYxX1VLgeuBBEZns0m8FLlXVMcDjQCQo/jNwtoj8DTgbqACauykrVt1vdgGrfN++fd1dqjFDjgUgMxgG4uesJ91xySJSAlwD3NHTglU1bqASkT0iUqKqla7svXHKqHDbrSKyGpgtImFgpqq+5rI9DfzR5duFawmJSCbwcVU9HK8sYEucz30UeBSgtLQ0XoA0xhjTTz1pCd0L/AnYrKp/FZFJwHv9/NwlwAK3vwD4XccMIpIjIkG3nw/MBTYAh4BsEZnqsl4AbIzkE5HINd0O/LibsowxA+zw4cP84Ac/6NO5Dz74IEeOHOl3HcrLy7nlllv6XU7EjTfeyLPPPtspfdeuXcyfPx+A1atXc/nll8c8f8KECezfv3/A6hOxcOFCNmzo+ldZvLpv376dp556qstzm5ubmT17dtzrGgjdBiFV/aWqnqqqX3Dvt6rqx/v5uWXABSLyHl7XXhmAiJSKyI9cnulAuYiswxtkUKaqG1S1CfhH4Ffu2KeA29w55wDviMi7ePeZvtFVWf28BmNMDEMhCJWWlvLQQw/1u5zujBo1KuYv+MHyox/9iBkzZvTp3J4Eoe9973tMnz69T+X3VLfdcSLyAHA/cBSv2+tU4FZV/VlfP1RVDwDnxUgvxxt0gKr+BTglzvm/AX4TI/1ZoNNPRFdlGXMsu+f369mwKzygZc4YFeKuvzsp7vFFixaxZcsWZs2axQUXXEBhYSHPPPMM9fX1XH311dxzzz3U1tZyzTXXsHPnTpqbm/m3f/s39uzZw65du/jYxz5Gfn4+q1atill+ZmYmn//851m6dCklJSV885vf5F/+5V/YsWMHDz74IFdccQWrV69m8eLFPPfcc9x9993s2LGDrVu3smPHDr7yla902Ur6yU9+wuLFixERTj31VH76058C8MILL/Cd73yH3bt388ADDzB//ny2b9/O5Zdfzttvv92ujAMHDnDddddRUVHBnDlz8G59x/btb3+bYDDILbfcwq233sq6detYuXIlK1eu5LHHHuPnP/85y5Yt46677qK+vp7Jkyfz+OOPk5mZyTnnnMPixYspLS3lscce4z/+4z8YOXIkM2fOJBgM8l//9V9x675o0SI2btzIrFmzWLBgAbfeemu7eu3cuZP/+7//44477uA73+nvWLT4etIdd6EbmHA5sB04gbaWhzHGtFNWVsbkyZN54403uOCCC3jvvfd4/fXXeeONN1izZg0vvPACf/zjHxk1ahTr1q3j7bff5uKLL+aWW25h1KhRrFq1Km4AAqitreXcc89l/fr1ZGVlceedd7J8+XJ+85vf8O///u8xz9m0aRN/+tOfeP3117nnnntobGyMmW/9+vXcf//9rFy5knXr1vG9732v9VhlZSUvvfQSzz33HIsWLeryO7jnnnv46Ec/yvr167n66qvZsWNH3Lzz5s3jxRdfBLxuxJqaGhobG3nxxRc566yz2L9/P/fffz/PP/88a9eupbS0tFNQ2LVrF/fddx+vvvoqL7/8Mps2bWp3PFbdy8rKmDdvHm+88Qa33noru3bt4tJLL2095ytf+QoPPPAAgUBi5zTo0cAEt70M+KWqVtnIG2OGh65aLINh2bJlLFu2jNmzZwNQU1PDe++9x7x58/ja177G17/+dS6//HLmzZvX4zJTU1O5+OKLATjllFMIBoOkpKRwyimnsH379pjnXHbZZQSDQYLBIIWFhezZs4cxY8Z0yrdy5Uo+8YlPkJ+fD0Bubm7rsauuuopAIMCMGTPYs2dPl3V84YUX+PWvf9362Tk5MSeFAeC0005jzZo1hMNhgsEgH/rQhygvL+fFF1/koYce4tVXX2XDhg3MnTsXgIaGBubMmdOujNdff52zzz67tb6f+MQnePfdd3tV91GjRrF06VIAnnvuOQoLCznttNNYvXp1l9faXz0JQs+JyCa87rjPi0gBYI9iG2O6parcfvvtfPazn+10bO3atSxdupQ777yT8847L24rpqOUlJTWocGBQIBgMNi639TUFPOcSB6ApKSkuPm6El1GV91rvZWSksLEiRN54oknOPPMMzn11FNZtWoVmzdvZvr06WzZsoULLriA//3f/+3zZ/S27i+//DJLlixh6dKl1NXVEQ6HueGGG/jZz/p8FyaungxMWAScCZSqaiNQizfjgTHGdJKVlUV1dTUAF110ET/+8Y+pqakBoKKigr1797Jr1y7S09O54YYbuO2221i7dm2nc/1w7rnn8stf/pIDBw4AcPDgwT6Vc9ZZZ7Xe9P/DH/7AoUOHusw/b948Fi9ezFlnncW8efP44Q9/yOzZsxERPvKRj/Dyyy+zefNmwOuOjG7lAHz4wx/mz3/+M4cOHaKpqYlf/epX3daxq+/6W9/6Fjt37mT79u384he/4Nxzz01IAIKezZiQAtwAPC0izwI3AQcSUhtjzLCXl5fH3LlzOfnkk1m+fDnXX389c+bM4ZRTTmH+/PlUV1fz1ltvcfrppzNr1izuuece7rzzTgBuvvlmLr74Yj72sY/5UveTTjqJO+64g7PPPpuZM2fy1a9+tfuTYrjrrrt44YUXOOmkk/j1r3/NuHHjusw/b948KisrmTNnDkVFRaSlpbV2URYUFPDEE09w3XXXceqppzJnzpxO93xGjx7Nv/7rv3L66aczd+5cJkyYQHZ2dpefeeqpp5KUlMTMmTP57ne/2+me0GCR7ppmbsh0Ct4cb+ANiW5W1YUJrtuQUFpaquXl5X5Xw5ge27hxY8KH1Zqhp6amhszMTJqamrj66qv5zGc+w9VXX53wz4318yYia9wMNd3qyT2hD6vqzKj3K93zNsYYY4aIu+++m+eff566ujouvPBCrrpqeEyR2ZMg1Cwik1V1C4CbMaG5m3OMMaZfzjjjDOrr69ul/fSnP+WUU/r/yN+BAwc477xOjyqyYsUK8vLyYpzRf4n+zMWLF/e7DD/0JAjdBqwSka2AAOOBTye0VsaY495rr73WfaY+ysvL44033khY+UPlM4eDboOQqq4QkSnAiS7pHcAWhzPGGNNvPVrUTlXrgdaF7ETkl0DXwz2MMcaYbvR1PgabMsEYY0y/9TUI2Ro7xhhj+i1uEBKR34vIkhiv3wOJGT5ijBn2hsJSDraeUJu+ric0YcIETjnlFGbNmkVpaY8e+emTru4JdTXeb3iOBTTGJFwkCH3hC1/o9bkPPvggN9xwA+np6f2qQ2lpaUJ/cUYMhfWE+ioShK6//vq4eVatWtU6mWuixA1CqvrnhH6yMSbx/rAIdr81sGUWnwKXlMU9bOsJHRvrCQ2WxC4UYYw57th6QsfGekIiwoUXXshpp53Go48+2uX19kePhmgngojkAk8DE/AWy7tGVTtNNSsizUDkT7kdqnqFSz8Xr1swFVgD3KSqTeLN8f494FLgCHCjqq515ywA7nRl3a+qT2LMsayLFstgsPWEhud6QgAvvfQSo0ePZu/evVxwwQVMmzaNs846q8vr7gvfghCwCFihqmUissi9/3qMfEdVtd3DsSISwJtQ9TxVfVdE7gUWAI8BlwBT3OsM4GHgDBf07gJK8Ub3rRGRJbECnzFmYNh6Qt0biusJgTczN0BhYSFXX301r7/+ekKCUE+Wcog1Su6nIvJlEUnrx2dfSdvM3E8CvZltLw9oUNVIqF8OfDyq3J+o51VgpIiUABcBy1X1oAs8y4GL+1F/Y0wMtp7Q8F9PqLa2tvVYbW0ty5Yt4+STT+62zL7oyT2hrUAN8D/uFQaqganufV8VqWql298NFMXJlyYi5SLyqohEAtV+IFlEIsNf5gNj3f5o4IOo83e6tHjpxpgBZOsJDf/1hPbs2cNHP/pRZs6cyemnn85ll13W2gU60HqyntBfVfXDsdJEZL2qxl3EXkSeB4pjHLoDeFJVR0blPaSqnTpORWS0qla42btX4nXBbRGROcADQBBYBlyuqrNE5DmgTFVfcuevwOvmOwdIU9X7Xfq/4XX1dRpuLiI3AzcDjBs37rT3338//hdkzBBj6wkdn47l9YQyRWScqu5whY8DMt2xhq5OVNXz4x0TkT0iUqKqla67bG+cMircdquIrAZmA1tU9RVgnivrQryWGUAFba0igDEurQIvEEWnr47zmY8Cj4K3qF1X12iMMUPBsbye0NeAl0RkC96ccROBL4hIBm33dPpiCd5ggjK3/V3HDCKSAxxR1XoRyQfm4rV+EJFCVd0rIkG8ls43osr9JxH5Bd7AhCoX6P4EfNOVCXAhcHs/6m+MSSBbT6h3jtn1hFR1qVvKYZpLekdV69z+g/347DLgGRG5CXgfuAbA3ef5nFs+fDrwiIi04N2/KlPVyBwVt4nI5S79YVVd6dKX4g3P3ow3RPvT7joOish9wF9dvntVtW93HY0xCWfrCR0fug1CIpICfBaIjM1bLSKPqGrsp716SFUPAJ3+LFDVcmCh2/8LEPPPHlW9DW/BvY7pCnwxzjk/Bn7c91obY4wZSD3pjnsYSAEiMxJ+yqUtTFSljDHGHB96EoQ+rKozo96vFJF1iaqQMcaY40dPnhOx2O8OAAAdB0lEQVRqFpHJkTduqHRz4qpkjDHmeNGTIHQbsEpEVovIn/Ge1flaYqtljBmubD2hzobjekKHDx9m/vz5TJs2jenTp/PKK6/0u76xdBuEVHUF3jxstwBfAk7EG3VmjDGdDIUgVFpaykMPPdTvcrozFNYTmjFjRp/O7S4IffnLX+biiy9m06ZNrFu3LmEPQPdoAlNVrQfejLwXkV8CXc9DYYzx3X+8/h9sOrip+4y9MC13Gl8/PdZcwx5bT2j4rydUVVXFCy+8wBNPPAF4M5enpqbGvYb+6Ot6QjKgtTDGHDNsPaHhv57Qtm3bKCgo4NOf/jSzZ89m4cKF1NbWdnnNfdXXpRxsKhtjhoGuWiyDwdYTGp7rCTU1NbF27Vq+//3vc8YZZ/DlL3+ZsrIy7rvvvi6vuy/iBiER+T2xg43gLaVgjDFdsvWEujcU1xMaM2YMY8aM4YwzzgBg/vz5lJUlZoHErrrjFgP/GeO1GG9aHGOM6cTWExr+6wkVFxczduxY3nnnHcCb366vAyC6E7clpKp/TsgnGmOOadHrCV1yySWt6wmBN6jgZz/7GZs3b+a2224jEAiQkpLCww8/DLStJxS5NzTYotcTSkpKYvbs2a0353vjrrvu4rrrruOkk07izDPP7NF6Qt/4xjeYM2cOGRkZcdcTikzoev/99zN16tTW86PXE8rNzWXatGm9Wk/oxhtv5B/+4R9YuHBha5fc97//fT75yU/S0NDApEmTePzxx3v9PfREt+sJHe9KS0u1vLzc72oY02O2ntDxabiuJ9TX0XHGGGOGkLvvvptZs2Zx8sknM3HixGNqPaFWIlKsqrsTVRljjImw9YR655hdT6iDpcCHElERY4yJZusJHR962x1nD6kaMwzYvV4zGAbi56y3Qeh/+v2JxpiESktL48CBAxaITEKpKgcOHCAtLa1f5fSqO05V+zYrYQcikgs8DUwAtgPXqGqngfQi0gy85d7uUNUrXPq5eM8rpQJrgJtUtUm8J9i+h/cc0xHgRlVd21VZxhxrxowZw86dO9m3b5/fVTHHuLS0tJgzT/RGX6ft6a9FwApVLRORRe59rPlFjqrqrOgEEQkATwLnqeq7InIvsAB4DLgEb8bvKcAZeCvAnhGvLGOORZEn8I0ZDvwaon0lXiDBbXszljAPaFDVyCPDy4GPR5X7E/W8CowUkZKBqLAxxpiB51cQKlLVSre/GyiKky9NRMpF5FURiQSq/UCyiEQehJoPjHX7o4EPos7f6dLilWWMMcZHCeuOE5HngeIYh+6IfqOqKiLx7qCOV9UKt6T4ShF5S1W3iMi1wHdFJAgso2fLjccsK07dbwZuBrqdbsMYY0zfJSwIqer58Y6JyB4RKVHVStddtjdOGRVuu1VEVgOzgS2q+gowz5V1IRCZRKmCtlYRwBiXFresOJ/7KPAoeNP29OR6jTHG9J5f3XFL8AYT4La/65hBRHJcSwcRyQfmAhvc+0K3DeINaPhhVLn/TzwfAapcoItbljHGGP/4NTquDHhGRG4C3geuAXD3eT6nqguB6cAjItKCFyzLVDUSOG4Tkctd+sOqutKlL8Ubnr0Zb4j2p116V2UZY4zxic2i3Q2bRdsYY3rHZtE2xhgzLFgQMsYY4xsLQsYYY3xjQcgYY4xvLAgZY4zxjQUhY4wxvrEgZIwxxjcWhIwxxvjGgpAxxhjfWBAyxhjjGwtCxhhjfGNByBhjjG8sCBljjPGNBSFjjDG+sSBkjDHGNxaEjDHG+MaCkDHGGN/4EoREJFdElovIe26bEydfs4i84V5LotLPFZG1IvK2iDwpIskufZqIvCIi9SLyzx3KulhE3hGRzSKyKLFXaIwxpif8agktAlao6hRghXsfy1FVneVeVwCISAB4ErhWVU8G3gcWuPwHgVuAxdGFiEgS8N/AJcAM4DoRmTHA12SMMaaX/ApCV+IFEtz2ql6cmwc0qOq77v1y4OMAqrpXVf8KNHY453Rgs6puVdUG4BeuDsYYY3zkVxAqUtVKt78bKIqTL01EykXkVRGJBKr9QLKIlLr384Gx3XzeaOCDqPc7XZoxxhgfJSeqYBF5HiiOceiO6DeqqiKicYoZr6oVIjIJWCkib6nqFhG5FviuiASBZUDzANf9ZuBmgHHjxg1k0cYYY6IkLAip6vnxjonIHhEpUdVKESkB9sYpo8Jtt4rIamA2sEVVXwHmubIuBKZ2U50K2reWxri0eHV/FHgUoLS0NF6ANMYY009+dcctoW0wwQLgdx0ziEiOa+kgIvnAXGCDe1/otkHg68APu/m8vwJTRGSiiKQC17o6GGOM8VHCWkLdKAOeEZGb8Ea3XQPg7vN8TlUXAtOBR0SkBS9YlqnqBnf+bSJyuUt/WFVXuvOLgXIgBLSIyFeAGaoaFpF/Av4EJAE/VtX1g3WxxhhjYhNV623qSmlpqZaXl/tdDWOMGTZEZI2qlnaf02ZMMMYY4yMLQsYYY3xjQcgYY4xvLAgZY4zxjQUhY4wxvrEgZIwxxjcWhIwxxvjGgpAxxhjfWBAyxhjjGwtCxhhj2gnXNfLO7upB+Sy/5o4zxhjjs+YWZdv+WjbtDrOpsppNu8NsrKym4vBRCrKC/PWOuIshDBgLQsYYcxw4VNvAxg7B5t091dQ3tQCQFBAmF2Rw2vgcPvmRcUwvDqGqiEhC62VByBhjjiGNzS1s3VfLxspwu6CzJ1zfmicvI5XpJSE+9ZHxTCsJMb0kixMKMwkmJw16fS0IGWPMMLW3uq41yGyqrGbj7mo2762msdlbHSElSTihMIu5k/OZXhJiWkkW04pDFGQFfa55GwtCCfJ333+J0IhkJuVnMqkgg0kFmUzKz2DUyBEkBRLbvDXGHFvqGpvZvLeGTbur2VQZ9ra7w+yvaWjNUxxKY1pJFmdPLWC6CzaTCjJISRra488sCCVAU3MLJxRmsnVfDb/9WwXV9U2tx1KTA0zMy3CBKYNJ+ZlMLMhgcn4m2ekpPtbaGOM3VWV32GvdbIgEm8owW/fX0tzitW6CyQFOLM7i3GmFTCv2WjfTi0PkZKT6XPu+sSCUAMlJAb77D7MA74dqf00DW/fVsHV/rbfdV8um3dUs27Cn9QcLvH7aSGCKtJ4m5mcwPi99yP81Y4zpnSMNTby7p6a1ZbPRbauONrbmGT1yBNNLsrjopOLW7rQJeRnHVG+KLyurikgu8DQwAdgOXKOqh2Lkawbecm93qOoVLv1cYDGQCqwBblLVJhGZBjwOfAi4Q1UXR5W1HagGmoGmnq76l8iVVRubW9hx8Ahb97UFp237a9m6v6ZdMzspIIzLTWdSfka74DSpIIOCzGDCR68YY/pOVdl56GhrkIncv9l2oJbIr9/01CSmFWd5gwTcdmpRFtkjhmfvSG9WVvUrCD0AHFTVMhFZBOSo6tdj5KtR1cwOaQHgfeA8VX1XRO4F3lfVx0SkEBgPXAUcihGESlV1f2/q6tfy3lVHGtm63wtMke22/d4rMqQSICuY3O6e08RIF19+BiNSB3+kizHHs+q6Rt7dU83GqGHQ7+yupsZ1yYvA+Nz01m60acXeyLSxOekEjqXWTS+CkF/dcVcC57j9J4HVQKcgFEce0KCq77r3y4HbgcdUdS+wV0QuG7iq+iM7PYXZ43KYPS6nXXpLi1Jx+Gi7rr1t+2t5besBfvO3inZ5R48c4br3MlzLyevmG5U94pj6gTdmsDW3KDsOHvFaN5VhNroWzgcHj7bmyUpLZnpxiL//0OjWYDO1KIuMoN0FiebXt1GkqpVufzdQFCdfmoiUA01Amar+FtgPJItIqaqWA/OBsT34TAWWiYgCj6jqo/27BH8EAsLY3HTG5qZz9tSCdseONDR53Xn7al1w8u5D/WptRetfYgBpKQEm5GUw2QWl6AAVShuezX9jEuXwkYZ2o9I27q7m3d3VHG1sBiAgMDE/g1PHjOTaD49r7VYblZ1mXeU9kLAgJCLPA8UxDt0R/UZV1QWGWMaraoWITAJWishbqrpFRK4FvisiQWAZ3n2e7nzUlVUILBeRTar6Qpy63wzcDDBu3LgeFD00pKcmc9KobE4ald0uXVXZV13PFte1t21fLVv317J+VxV/XL+73eCI/MwgkwoymBwJTm6QxNhcGxxhjm1NzS1s21/rtWoqw633cCqr6lrz5KSnML0kxHWnj2sdlTalKJO0FOv67quEBSFVjTvpkIjsEZESVa0UkRJgb5wyKtx2q4isBmYDW1T1FWCeK+tCYGoP6hMpa6+I/AY4HYgZhFwr6VHw7gl1V/ZQJyIUhtIoDKUxZ3Jeu2MNTS3sOFjLlsigCNfF96f1ezhY2zY4IjkgjMtLbxu5F9V6ystItb/4zLByoKa+dURa5P7Ne3traHD3W5MDwgmFmZwxMZdpJSGmFWcxvSREYZYNBBpofnXHLQEWAGVu+7uOGUQkBziiqvUikg/MBR5wxwpdMAni3Uv6RlcfJiIZQEBVq93+hcC9A3lBw1VqcoATCrM4oTCr07HDRxo6Baet+2t44d19NDS3DY4IpSW3BqTo4DQhL8P+QjS+amhqcQ95th8Gva+6bQqbgqwg04qzuPHMCa3BZnJBJqnJ1vIfDH6NjssDngHG4Y10u0ZVD4pIKfA5VV0oImcCjwAteEtOPKiqj7nzvw1c7tIfVtUHXXoxUA6E3Hk1wAwgH/iN+/hk4ClV7TJwRfg1Om4oa25RKg4d7TR6b+u+WnaH27ouRCKDIzLbhpe7llRxKM0GR5gBo6rsra5vGwbttpv31tDkuptTkwNMLcr0Rqa5YHNicRb5mUNnCptjxZAfoj2cWBDqndp6NziiQ+tp275aahvabt2NSElqfdYpuvU0MT+DLBscYbpQ19jMe3tq2Ljb3bdx3WmHjrQ95DkqO621Gy3y7M3E/AyS7b7moBgOQ7TNMSojmMzJo7M5eXTnwRF7q+vZsq+t1bR1fw1v7qxi6VuVRI2NoDAr2Dpib3LU9EZjckbYL5HjiKr3OELr0gOuhbNtf23rz0taSoATi0NcdFJxa+tmWnHIpsAaRiwImUEhIhSF0igKpXHm5Px2x+qbmtlx4Ejr6L3IDBJ/eLuSw1F/3aYkCePzMlpbUJOjpjfKHabzZhlPbX0T7+yp7jAjdJjqurZHC8bmjmBacYjLTh3VOqvAuNz0Y2oKm+ORBSHju2ByElOKsphS1HlwxKHaBrbur/ECVGR6o/21rH5nb+t09QAj01PaDSmf7ILT+Lx0X9ZIMbG1tCgfHDoSNaOAd+/m/QNHWvNkBpOZVpzFlbNGtXvI07ppj012T6gbdk9oaGpqbvFmjthX63XxuXtQ2/bXtlu8KyAwOmdEu0lhJ7uuvqKQDbdNpHBdY7tVPDftDvPO7mqOuHuDIjAxL6N1+ppId9qYnBH27zLM2T0hc8xLTgowPi+D8XkZfGxaYbtjNfVN7oHcSAvKC06vbzvY+pQ7eJNGts4WEenic5PD2tQqPdfcomzbX9vajRYJOhWH26awyR6RwrTiLK4pHdsabKYUZZKeat/z8c5aQt2wltCxI7JWS6Rbb0vUrOU7Dx0l+r9CUSjYfkFCdw9qdM7xvSjhodqGdktGb6ys5t091a2T6iYFhEn5Ga3LDkx3E3UWh2wKm+OJtYSMiUFEKMkeQUn2COae0H5wRF1jM+8fOBK17pMXnJ57s7Ld+i6pSQHG56W3m7k8sh2ui4rF0tjc4ta9CnuLq7mgE93VmZeRyvSSEJ/6yPjW4dAnFNoUNqZ3LAgZA6SlJHFicRYnFrcfHKGqHKxtaHvuyQWozXtrWLmp/eCInPSU9oHJDZAYl5sxpJ++31td12FUWjWb91a3XltKknBCYRZzJ+e33b8pyaIwK83nmptjgQUhY7ogIuRlBsnLDPLhCbntjjU1t/DBoaOt95wi959Wv7uPX67Z2ZovIDC2dVHCzHYzRwzmXGR1jc1uCpu2GQU27Q63W0CxKBRkWnGIs6cWMN0FnEkFGTZ5rUkYC0LG9FFyUoCJbq2mjsJ1ja2DI7btq2WLa0G9svUAdY1t8+5lBpOjZo6I3IPyyuzrTfvIva9NldVeV5oLOlv317bOmB5MDnBicRbnTitst8CaPW9lBpsFIWMSIJSWwsyxI5k5dmS79JYWpTJc19p6igwxL99+iCXrdrUbHFGSndYhOHldfaNGtg2OONrQ7B7ybD9BZ/R9rNEjRzC9JMubVcAFm4n5Gcf1AAszdFgQMmYQBQLC6JEjGD1yBPOmtF+UsK6xuXUJ98i8e1v21/LbNyrazRyQmhxgYl4Gjc0tbDtQ2xq40lO9+1qXnlLC9BJvGPTUoiyyR9hDnmbosiBkzBCRlpLE9JIQ00tC7dJVlf01DW3ByW2TAsIVUbMKjM1Jt5nJzbBjQciYIU5EKMgKUpAV5PSJud2fYMwwYkNejDHG+MaCkDHGGN9YEDLGGOMbC0LGGGN840sQEpFcEVkuIu+5bU6cfM0i8oZ7LYlKP1dE1orI2yLypIgku/RPisibIvKWiPxFRGZGnXOxiLwjIptFZFHir9IYY0x3/GoJLQJWqOoUYIV7H8tRVZ3lXlcAiEgAeBK4VlVPBt4HFrj824CzVfUU4D7gUXdOEvDfwCXADOA6EZmRmEszxhjTU34FoSvxAglue1Uvzs0DGlT1Xfd+OfBxAFX9i6oecumvAmPc/unAZlXdqqoNwC9cHYwxxvjIryBUpKqVbn83UBQnX5qIlIvIqyISCVT7gWQRiaxVMR8YG+Pcm4A/uP3RwAdRx3a6tJhE5Gb3ueX79u3rweUYY4zpi4Q9rCoizwPFMQ7dEf1GVVVE4q2sN15VK0RkErBSRN5S1S0ici3wXREJAsuA5uiTRORjeEHoo32pu6o+SltX3j4Reb8v5QD5eEFzqLF69Y7Vq3esXr1zLNZrfE8zJiwIqer58Y6JyB4RKVHVShEpAfbGKaPCbbeKyGpgNrBFVV8B5rmyLgSmRpV9KvAj4BJVPeCSK2jfWhrj0npyHQXd54pNRMp7urrgYLJ69Y7Vq3esXr1zvNfLr+64JbQNJlgA/K5jBhHJcS0dRCQfmAtscO8L3TYIfB34oXs/Dvg18Kmoe0YAfwWmiMhEEUkFrnV1MMYY4yO/glAZcIGIvAec794jIqUi8iOXZzpQLiLrgFVAmapucMduE5GNwJvA71V1pUv/d7yBCz9ww7rLAVS1Cfgn4E/ARuAZVV2f8Ks0xhjTJV8mMHXdZOfFSC8HFrr9vwCnxDn/NuC2GOkLI+fHOLYUWNr3WvfJo4P8eT1l9eodq1fvWL1657iul6jGGxNgjDHGJJZN22OMMcY3FoT6SESSRORvIvJcjGNBEXnaTRH0mohMiDp2u0t/R0QuGuR6fVVENripjVaIyPioYzGnSBqket3ohsJHPn9h1LEFbnqn90RkQcdzE1yv70bV6V0RORx1LNHf13Y3/VTrvc0Ox0VEHnI/S2+KyIeijiXsO+tBvbqaOqvLcxNcr3NEpCrq3+zfo44lbEqvHtTrtqg6ve1+rnJ7cm4/6zVSRJ4VkU0islFE5nQ4Png/X6pqrz68gK8CTwHPxTj2BeCHbv9a4Gm3PwNYBwSBicAWIGkQ6/UxIN3tfz5SL/e+xsfv60bgv2Kk5wJb3TbH7ecMVr065PsS8ONB/L62A/ldHL8U72FsAT4CvDYY31kP6nVm5PPwpsl6rafnJrhe58T52Uty/w8nAanu/+eMwapXh7x/B6wcpO/rSWCh208FRvr182UtoT4QkTHAZXjPI8USPS3Rs8B5IiIu/ReqWq+q24DNeFMKDUq9VHWVqh5xb6OnNUqoHnxf8VwELFfVg+pNx7QcuNinel0H/O9AffYAuBL4iXpeBUaK98xdQr+z7mj8qbOGqqE0pdeg/IyJSDZwFvAYgKo2qOrhDtkG7efLglDfPAj8C9AS53jrNEHqDQ+vwhs63qvpgxJQr2jR0xpB7CmSBrNeH3fN/mdFJPJg8ZD4vly35URgZVRyIr8vAAWWicgaEbk5xvF4302iv7Pu6hWt489Yb85NRL3miMg6EfmDiJzk0obE9yUi6Xi/zH/V23P7YCKwD3jcdUX/SEQyOuQZtJ8vX4ZoD2cicjmwV1XXiMg5ftcnojf1EpEbgFLg7Kjk8RpjiqRBqtfvgf9V1XoR+SxeK/Lc/n72ANQr4lrgWVWNnh4qId9XlI+68guB5SKySVVfGMDy+6pH9ZLYU2cl8pq6K3st3r9ZjYhcCvwWmDJAn92fekX8HfCyqh7sw7m9lQx8CPiSqr4mIt/DW8ng3wag7F6zllDvzQWuEJHteE33c0XkZx3ytE4TJN5aR9nAAfoxfdAA1QsROR9v/r4rVLU+kq5RUyQBq/GmSBqUeqnqgai6/Ag4ze37/n0519KhmySB31fH8vcCv6Fzt2287yaR31lP6hU9ddaV2jZ1Vo/OTVS9VDWsqjVufymQIt5MLL5/X05XP2MD/X3tBHaq6mvu/bN4QSna4P18JeKm1/HyIv7Nzi/SfmDCM27/JNoPTNjKAA9M6KZes/Fuwk7pkJ4DBN1+PvAeA3hztgf1Konavxp41e3n4q0RleNe24DcwaqXOzYN7waxDNb3BWQAWVH7fwEu7pDnMtrfOH490d9ZD+s1Du9e55m9PTfB9SqO/Bvi/TLf4b67ZPf/cCJtAxNOGqx6uWPZwEEgYzC+L1fmi8CJbv9u4Nt+/XxZd9wAEZF7gXJVXYJ3w++nIrIZ74frWgBVXS8iz+DNgdcEfFHbd/Ekul7fBjKBX3rjJNih3mKB04FHRKQFr3UcPUXSYNTrFhG5Au87OYg3Wg5VPSgi9+HN/Qdwr7bvrkh0vcD7t/uFuv+BTqK/ryLgN+7fKBl4SlX/KCKfA1DVH+LN/nEp3i/8I8Cn3bFEfmc9qVf01FkATepNghnz3EGs13zg8yLSBBzFWxRTgSYRiUzplYQ3AnKgpvTqSb3A+8NrmarWdnfuANULvNGePxdvLs2twKf9+vmyGROMMcb4xu4JGWOM8Y0FIWOMMb6xIGSMMcY3FoSMMcb4xoKQMcYY31gQMsYY4xsLQsYcA8Sb9j+/j+feKCKjBqIsY3rLgpAx5kZgVHeZjEkEC0LGDCARmSDeQmFPiLcQ3s9F5HwRedktAna6e73iZjD+i4ic6M69VUR+7PZPEW+Rs/Q4n5MnIstEZL2I/AhvepXIsRtE5HXxFkN7RESSXHqNeAv1rRdvUcMCEZmPN5ntz13+Ea6YL4nIWvEWVZuWyO/MHN8sCBkz8E4A/hNv3rlpwPV4s0n/M/CvwCZgnqrOxpvm5pvuvO8BJ4jI1cDjwGe1bf2nju4CXlLVk/AmtxwHICLTgX8A5qrqLKAZ+KQ7JwNvSqKTgD8Dd6nqs0A58ElVnaWqR13e/ar6IeBhV29jEsLmjjNm4G1T1bcARGQ9sEJVVUTeAibgTVj5pIhMwVszJgVAVVtE5EbgTeARVX25i884C/h7d97/iUhkIbnz8GYh/6ubd2wEsNcdawGedvs/A37dRfmRY2sin2NMIlgQMmbg1Uftt0S9b8H7P3cfsEpVrxaRCXhLQURMAWro+z0aAZ5U1dt7kLeriSMjdW7Gfk+YBLLuOGMGXzZta7DcGEkUb9nlh/BaOXnufk08L+B18yEil+BNqw+wApjvFkJDRHLFWxkWvP/vkTKvB15y+9VAVj+ux5g+syBkzOB7APiWiPyN9q2M7wL/rarv4q1KWhYJJjHcA5zluvv+Hm99HNySEnfiLQv9JrAcKHHn1AKni8jbeCvX3uvSnwB+2GFggjGDwpZyMOY4ISI1qprpdz2MiWYtIWOMMb6xlpAxQ5iIfBr4cofkl1X1i37Ux5iBZkHIGGOMb6w7zhhjjG8sCBljjPGNBSFjjDG+sSBkjDHGNxaEjDHG+Ob/A45kRLa4kJgCAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch2_2.best_score_, gsearch2_2.best_params_))\n",
    "test_means = gsearch2_2.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch2_2.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch2_2.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch2_2.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch2_2.cv_results_).to_csv('my_preds_maxdepth_min_child_weights_2.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(min_child_weight), len(max_depth))\n",
    "train_scores = np.array(train_means).reshape(len(min_child_weight), len(max_depth))\n",
    "\n",
    "for i, value in enumerate(min_child_weight):\n",
    "    pyplot.plot(max_depth, test_scores[i], label= 'test_min_child_weight:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'max_depth' )                                                                                                      \n",
    "pyplot.ylabel( '- Log Loss' )\n",
    "pyplot.savefig( 'max_depth_vs_min_child_weght2.png' )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "根据第二次的调试，最佳参数组合：{'max_depth': 5, 'min_child_weight': 5}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0, 1)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb2_2.reg_alpha,xgb2_2.reg_lambda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
